From 2b544b7ba0e9f03c6b98c7a4c58d799b95e72b92 Mon Sep 17 00:00:00 2001 From: MOVZX Date: Mon, 19 Oct 2026 02:19:03 +0700 Subject: [PATCH] [Mini-VLLM] Cut dead subsystems across serving, connectors, parsers - Remove P/D transfer connectors: nixl, nixl_chat, mooncake x2, hf3fs, moriio, hisparse, example and hidden-states connectors - Reduce weight_transfer to the base API used by model loading - Remove kv_offload tiering, simple_kv_offload, ec cpu/example, snapshot command, engine snapshot server - Remove transformers model fallback, dsv4 tokenizers, dead platforms cpu/tpu/xpu, flashmla third_party, dsa, megamoe/flydsl, dead attn ops, dead layers (short_conv, fused_embed_norm, indexer_topk, relu2 gemma_ar, lightning_attn, minimax_rms_norm), gemm_rs_ar - Keep MoE LoRA op (imported eagerly by punica_gpu); drop fp8 variant - Rebuild parser hubs around Qwen3 adapters only (36 dead adapters) - Remove dead tests with the code they covered Co-authored-by: Hinata Signed-off-by: MOVZX --- tests/distributed/test_packed_tensor.py | 775 --- tests/distributed/test_sharded_rdt_trainer.py | 628 --- tests/distributed/test_weight_transfer.py | 2399 -------- .../test_weight_transfer_nccl_uid.py | 465 -- .../test_across_api_servers_metrics.py | 70 - .../attention/test_pack_unpack_triton.py | 234 - .../generation/vlm_utils/model_utils.py | 246 +- tests/models/qwen4_exp/test_ple.py | 436 -- .../models/transformers/fusers/test_linear.py | 1439 ----- tests/models/transformers/fusers/test_mla.py | 181 - tests/models/transformers/fusers/test_moe.py | 611 --- .../transformers/fusers/test_rms_norm.py | 409 -- .../models/transformers/test_dynamic_rope.py | 86 - .../transformers/test_layer_registry.py | 80 - tests/multimodal/media/test_image.py | 14 - tests/parser/cohere/test_citations.py | 172 - tests/parser/cohere/test_structural_tags.py | 476 -- tests/parser/engine/test_delegating_replay.py | 211 - tests/parser/engine/test_replay.py | 632 --- .../engine/test_ufffd_reasoning_transition.py | 23 +- tests/parser/engine/trace_builder.py | 1358 ----- tests/platforms/test_cpu.py | 224 - .../test_base_thinking_reasoning_parser.py | 436 -- tests/test_pcp_dp.py | 161 - .../processors/test_pixtral.py | 65 - .../processors/test_voxtral.py | 127 - .../test_bailing_moe_v3_vl_config.py | 159 - .../test_dspark_mla_config.py | 153 - tests/v1/core/test_prefix_caching.py | 1423 +---- tests/v1/core/test_scheduler.py | 69 - .../unit/test_bidirectional_kv_transfer.py | 1082 ---- .../v1/kv_connector/unit/test_hf3fs_client.py | 283 - .../kv_connector/unit/test_hf3fs_connector.py | 233 - .../unit/test_hf3fs_metadata_server.py | 192 - .../kv_connector/unit/test_hisparse_stats.py | 158 - .../unit/test_moriio_hma_scheduler.py | 1414 ----- .../unit/test_moriio_proxy_routing.py | 335 -- .../kv_connector/unit/test_moriio_routing.py | 98 - .../unit/test_moriio_routing_fairness.py | 210 - .../kv_connector/unit/test_moriio_tp_ack.py | 494 -- .../v1/kv_connector/unit/test_moriio_unmap.py | 68 - .../unit/test_moriio_write_done_routing.py | 449 -- .../kv_connector/unit/test_nixl_connector.py | 4820 ----------------- .../kv_connector/unit/test_nixl_heartbeat.py | 253 - .../unit/test_nixl_push_connector.py | 2156 -------- .../unit/test_remote_prefill_lifecycle.py | 977 ---- .../unit/test_simple_cpu_offload_connector.py | 149 - .../unit/test_simple_cpu_offload_metrics.py | 466 -- tests/v1/kv_connector/unit/test_tp_mapping.py | 316 -- tests/v1/kv_connector/unit/utils.py | 148 - tests/v1/simple_kv_offload/test_scheduler.py | 3395 ------------ vllm/config/vllm.py | 13 +- .../ec_transfer/ec_connector/factory.py | 18 - .../kv_transfer/kv_connector/factory.py | 47 - .../kv_connector/v1/hf3fs/__init__.py | 0 .../kv_connector/v1/hf3fs/hf3fs_client.py | 302 -- .../kv_connector/v1/hf3fs/hf3fs_connector.py | 1196 ---- .../v1/hf3fs/hf3fs_metadata_server.py | 532 -- .../kv_connector/v1/hf3fs/utils/__init__.py | 0 .../kv_connector/v1/hf3fs/utils/common.py | 139 - .../v1/hf3fs/utils/gather_scatter_helper.py | 262 - .../v1/hf3fs/utils/hf3fs_mock_client.py | 133 - .../v1/hf3fs/utils/hf3fs_utils.cpp | 57 - .../kv_connector/v1/hisparse/__init__.py | 2 - .../kv_connector/v1/hisparse/connector.py | 408 -- .../kv_connector/v1/hisparse/stats.py | 214 - .../kv_connector/v1/hisparse/worker.py | 978 ---- .../kv_connector/v1/moriio/__init__.py | 0 .../kv_connector/v1/moriio/moriio_common.py | 656 --- .../v1/moriio/moriio_connector.py | 3856 ------------- .../kv_connector/v1/moriio/moriio_engine.py | 1084 ---- .../v1/moriio/moriio_heartbeat.py | 96 - .../kv_connector/v1/moriio/moriio_layout.py | 615 --- .../kv_connector/v1/nixl/__init__.py | 61 - .../kv_connector/v1/nixl/base_scheduler.py | 549 -- .../kv_connector/v1/nixl/base_worker.py | 3922 -------------- .../kv_connector/v1/nixl/connector.py | 426 -- .../kv_connector/v1/nixl/metadata.py | 340 -- .../kv_connector/v1/nixl/pull_scheduler.py | 292 - .../kv_connector/v1/nixl/pull_worker.py | 693 --- .../kv_connector/v1/nixl/push_scheduler.py | 357 -- .../kv_connector/v1/nixl/push_worker.py | 827 --- .../kv_connector/v1/nixl/scheduler.py | 12 - .../kv_transfer/kv_connector/v1/nixl/stats.py | 328 -- .../kv_connector/v1/nixl/tp_mapping.py | 161 - .../kv_transfer/kv_connector/v1/nixl/utils.py | 67 - .../kv_connector/v1/nixl/worker.py | 13 - .../v1/simple_cpu_offload_connector.py | 386 -- .../v1/ssm_conv_transfer_utils.py | 235 - .../distributed/weight_transfer/ipc_engine.py | 459 -- .../distributed/weight_transfer/m2n_common.py | 864 --- .../distributed/weight_transfer/m2n_engine.py | 599 -- .../distributed/weight_transfer/m2n_layout.py | 261 - .../distributed/weight_transfer/m2n_source.py | 207 - .../weight_transfer/m2n_trainer.py | 628 --- .../weight_transfer/modelexpress_engine.py | 41 - .../weight_transfer/nccl_common.py | 299 - .../weight_transfer/nccl_engine.py | 461 -- .../weight_transfer/packed_tensor.py | 494 -- .../weight_transfer/sharded_rdt_common.py | 352 -- .../weight_transfer/sharded_rdt_engine.py | 1477 ----- .../weight_transfer/sharded_rdt_fake.py | 300 - .../weight_transfer/sharded_rdt_trainer.py | 1520 ------ .../weight_transfer/sparse_nccl_engine.py | 377 -- vllm/envs.py | 4 - vllm/lora/ops/torch_ops/__init__.py | 20 - vllm/lora/ops/torch_ops/lora_ops.py | 128 - vllm/lora/ops/triton_ops/__init__.py | 26 +- .../ops/triton_ops/fused_moe_lora_fp8_op.py | 895 --- .../lora/ops/triton_ops/lora_expand_fp8_op.py | 377 -- .../lora/ops/triton_ops/lora_shrink_fp8_op.py | 404 -- vllm/lora/punica_wrapper/punica_cpu.py | 371 -- .../kernels/attention/dsa/__init__.py | 2 - .../kernels/attention/dsa/candidate_blocks.py | 271 - .../attention/dsa/sparse_mqa_logits.py | 498 -- .../kernels/linear/cute_dsl/gemm_rs_ar.py | 1177 ---- .../model_executor/layers/fused_embed_norm.py | 231 - .../layers/fused_moe/fused_flydsl_moe.py | 391 -- .../layers/fused_moe/prepare_megamoe.py | 682 --- .../layers/fusion/relu2_fp8_quant.py | 76 - vllm/model_executor/layers/indexer_topk.py | 405 -- vllm/model_executor/layers/lightning_attn.py | 741 --- .../model_executor/layers/mamba/short_conv.py | 382 -- .../layers/minimax_rms_norm/__init__.py | 10 - .../minimax_rms_norm/lamport_workspace.py | 316 -- .../layers/minimax_rms_norm/rms_norm_tp.py | 421 -- vllm/model_executor/models/registry.py | 72 +- .../models/transformers/__init__.py | 155 - .../models/transformers/base.py | 860 --- .../models/transformers/causal.py | 89 - .../models/transformers/fuser.py | 126 - .../models/transformers/fusers/__init__.py | 33 - .../models/transformers/fusers/attention.py | 162 - .../models/transformers/fusers/base.py | 250 - .../models/transformers/fusers/glu.py | 191 - .../transformers/fusers/merged_column.py | 145 - .../models/transformers/fusers/mla.py | 303 -- .../models/transformers/fusers/moe.py | 439 -- .../models/transformers/fusers/packed_qkv.py | 164 - .../models/transformers/fusers/qkv.py | 209 - .../models/transformers/fusers/rms_norm.py | 297 - .../models/transformers/fx_utils.py | 818 --- .../models/transformers/layers.py | 64 - .../models/transformers/legacy.py | 80 - .../model_executor/models/transformers/moe.py | 454 -- .../models/transformers/multimodal.py | 1267 ----- .../models/transformers/utils.py | 407 -- vllm/model_executor/warmup/kernel_warmup.py | 33 +- vllm/models/qwen4_exp/amd/__init__.py | 2 - vllm/models/qwen4_exp/amd/hyperconnection.py | 205 - vllm/models/qwen4_exp/amd/indexer_qsa.py | 420 -- vllm/models/qwen4_exp/amd/low_latency_gemm.py | 14 - vllm/models/qwen4_exp/amd/model.py | 1097 ---- vllm/models/qwen4_exp/amd/model_state.py | 143 - vllm/models/qwen4_exp/amd/mtp.py | 495 -- vllm/models/qwen4_exp/amd/ops/__init__.py | 3 - vllm/models/qwen4_exp/amd/ops/hc.py | 487 -- vllm/models/qwen4_exp/amd/ops/qsa.py | 1122 ---- vllm/models/qwen4_exp/amd/ops/qsa_prepare.py | 703 --- .../qwen4_exp/amd/ops/sequence_parallel.py | 47 - vllm/models/qwen4_exp/amd/ple_layer.py | 883 --- vllm/models/qwen4_exp/amd/qsa.py | 544 -- vllm/parser/AGENTS.md | 41 - vllm/parser/beam.py | 262 - vllm/parser/cohere_command.py | 691 --- vllm/parser/deepseek_v32.py | 144 - vllm/parser/deepseek_v4.py | 277 - vllm/parser/deepseek_v41.py | 54 - vllm/parser/engine/registered_adapters.py | 102 - vllm/parser/gemma4.py | 636 --- vllm/parser/glm47_moe.py | 286 - vllm/parser/granite.py | 88 - vllm/parser/granite_thinking.py | 131 - vllm/parser/inkling.py | 332 -- vllm/parser/kimi_k2.py | 268 - vllm/parser/kimi_k3.py | 135 - vllm/parser/ling3.py | 80 - vllm/parser/mimo.py | 47 - vllm/parser/minicpmv.py | 405 -- vllm/parser/minimax_m2.py | 265 - vllm/parser/minimax_m3.py | 293 - vllm/parser/nemotron_v3.py | 113 - vllm/parser/plamo3.py | 318 -- vllm/parser/seed_oss.py | 31 - vllm/parser/step3p5.py | 49 - vllm/platforms/__init__.py | 68 - vllm/platforms/cpu.py | 797 --- vllm/platforms/zen_cpu.py | 34 - vllm/reasoning/AGENTS.md | 54 - vllm/reasoning/basic_parsers.py | 196 - vllm/reasoning/beam_reasoning_parser.py | 6 - .../cohere_command_reasoning_parser.py | 60 - .../reasoning/deepseek_r1_reasoning_parser.py | 66 - .../reasoning/deepseek_v3_reasoning_parser.py | 96 - .../deepseek_v41_engine_reasoning_parser.py | 6 - .../deepseek_v4_engine_reasoning_parser.py | 6 - vllm/reasoning/ernie45_reasoning_parser.py | 157 - .../gemma4_engine_reasoning_parser.py | 6 - vllm/reasoning/gemma4_utils.py | 131 - vllm/reasoning/glm47_moe_reasoning_parser.py | 6 - vllm/reasoning/granite_reasoning_parser.py | 372 -- ...ranite_thinking_engine_reasoning_parser.py | 8 - .../hunyuan_a13b_reasoning_parser.py | 226 - vllm/reasoning/hy_v3_reasoning_parser.py | 143 - vllm/reasoning/hy_v4_reasoning_parser.py | 323 -- vllm/reasoning/identity_reasoning_parser.py | 72 - vllm/reasoning/inkling_reasoning_parser.py | 6 - vllm/reasoning/k2_horizon_reasoning_parser.py | 159 - vllm/reasoning/kimi_k2_reasoning_parser.py | 8 - vllm/reasoning/kimi_k3_reasoning_parser.py | 468 -- vllm/reasoning/ling3_reasoning_parser.py | 6 - .../reasoning/mimo_engine_reasoning_parser.py | 6 - vllm/reasoning/minicpmv_reasoning_parser.py | 16 - vllm/reasoning/minimax_m2_reasoning_parser.py | 61 - vllm/reasoning/minimax_m3_reasoning_parser.py | 320 -- vllm/reasoning/mistral_reasoning_parser.py | 6 - .../muse_glimmer_reasoning_parser.py | 327 -- .../nemotron_v3_engine_reasoning_parser.py | 8 - vllm/reasoning/olmo3_reasoning_parser.py | 317 -- .../plamo3_engine_reasoning_parser.py | 6 - .../reasoning/poolside_v1_reasoning_parser.py | 71 - .../response_template_reasoning_parser.py | 6 - .../seed_oss_engine_reasoning_parser.py | 6 - vllm/reasoning/step3_reasoning_parser.py | 128 - .../step3p5_engine_reasoning_parser.py | 6 - vllm/third_party/flashmla/__init__.py | 1 - vllm/tokenizers/deepseek_v32.py | 81 - vllm/tokenizers/deepseek_v32_encoding.py | 312 -- vllm/tokenizers/deepseek_v4.py | 103 - vllm/tokenizers/deepseek_v41_encoding.py | 577 -- vllm/tokenizers/deepseek_v4_encoding.py | 648 --- vllm/tokenizers/registry.py | 4 - vllm/tool_parsers/AGENTS.md | 45 - vllm/tool_parsers/__init__.py | 1 - vllm/tool_parsers/apertus_tool_parser.py | 553 -- vllm/tool_parsers/beam_tool_parser.py | 6 - .../cohere_command_tool_parser.py | 18 - vllm/tool_parsers/deepseekv31_tool_parser.py | 392 -- .../deepseekv32_engine_tool_parser.py | 8 - vllm/tool_parsers/deepseekv3_tool_parser.py | 395 -- .../deepseekv41_engine_tool_parser.py | 8 - .../deepseekv4_engine_tool_parser.py | 8 - vllm/tool_parsers/dots_tool_parser.py | 434 -- vllm/tool_parsers/ernie45_tool_parser.py | 211 - .../tool_parsers/functiongemma_tool_parser.py | 313 -- .../tool_parsers/gemma4_engine_tool_parser.py | 41 - vllm/tool_parsers/gemma4_utils.py | 158 - vllm/tool_parsers/gigachat3_tool_parser.py | 214 - vllm/tool_parsers/glm47_moe_tool_parser.py | 14 - vllm/tool_parsers/granite4_tool_parser.py | 256 - .../granite_20b_fc_tool_parser.py | 275 - .../granite_engine_tool_parser.py | 8 - vllm/tool_parsers/hermes_tool_parser.py | 274 - vllm/tool_parsers/hunyuan_a13b_tool_parser.py | 415 -- vllm/tool_parsers/hy_v3_tool_parser.py | 697 --- vllm/tool_parsers/hy_v4_tool_parser.py | 1174 ---- vllm/tool_parsers/inkling_tool_parser.py | 11 - vllm/tool_parsers/internlm2_tool_parser.py | 248 - vllm/tool_parsers/jamba_tool_parser.py | 337 -- vllm/tool_parsers/k2_horizon_tool_parser.py | 462 -- vllm/tool_parsers/kimi_k2_tool_parser.py | 20 - vllm/tool_parsers/kimi_k3_tool_parser.py | 399 -- vllm/tool_parsers/lfm2_tool_parser.py | 434 -- vllm/tool_parsers/ling3_tool_parser.py | 11 - .../llama4_pythonic_tool_parser.py | 210 - vllm/tool_parsers/llama_tool_parser.py | 334 -- vllm/tool_parsers/longcat_tool_parser.py | 24 - vllm/tool_parsers/mimo_tool_parser.py | 8 - vllm/tool_parsers/minicpm5xml_tool_parser.py | 815 --- vllm/tool_parsers/minimax_m2_tool_parser.py | 8 - vllm/tool_parsers/minimax_m3_tool_parser.py | 10 - vllm/tool_parsers/mistral_tool_parser.py | 49 - vllm/tool_parsers/muse_glimmer_tool_parser.py | 506 -- vllm/tool_parsers/olmo3_tool_parser.py | 227 - vllm/tool_parsers/phi4mini_tool_parser.py | 123 - .../tool_parsers/plamo3_engine_tool_parser.py | 9 - vllm/tool_parsers/poolside_v1_tool_parser.py | 610 --- vllm/tool_parsers/pythonic_tool_parser.py | 199 - .../response_template_tool_parser.py | 6 - .../seed_oss_engine_tool_parser.py | 8 - vllm/tool_parsers/step3_tool_parser.py | 306 -- vllm/tool_parsers/step3p5_tool_parser.py | 8 - vllm/tool_parsers/xlam_tool_parser.py | 561 -- vllm/transformers_utils/configs/__init__.py | 62 +- .../configs/bailing_moe_v3_vl.py | 345 -- vllm/transformers_utils/configs/chatglm.py | 75 - .../configs/colmodernvbert.py | 63 - vllm/transformers_utils/configs/colpali.py | 56 - .../configs/cosmos3_edge.py | 147 - .../configs/deepseek_v41.py | 62 - .../configs/deepseek_vl2.py | 126 - vllm/transformers_utils/configs/dots3_note.py | 25 - vllm/transformers_utils/configs/falcon.py | 89 - .../configs/funaudiochat.py | 120 - vllm/transformers_utils/configs/hy_v4.py | 173 - vllm/transformers_utils/configs/isaac.py | 99 - vllm/transformers_utils/configs/k3_dspark.py | 77 - vllm/transformers_utils/configs/kimi_k3.py | 139 - .../transformers_utils/configs/kimi_linear.py | 161 - vllm/transformers_utils/configs/kimi_vl.py | 38 - .../transformers_utils/configs/midashenglm.py | 103 - vllm/transformers_utils/configs/minimax_m3.py | 149 - vllm/transformers_utils/configs/moondream3.py | 153 - vllm/transformers_utils/configs/moonvit.py | 33 - .../configs/moss_transcribe_diarize.py | 108 - .../configs/muse_glimmer.py | 389 -- vllm/transformers_utils/configs/openvla.py | 50 - vllm/transformers_utils/configs/ovis.py | 124 - .../transformers_utils/configs/qianfan_ocr.py | 105 - vllm/transformers_utils/configs/qwen3_asr.py | 455 -- vllm/transformers_utils/configs/radio.py | 110 - vllm/transformers_utils/configs/step3_vl.py | 178 - vllm/transformers_utils/configs/step3p5.py | 103 - vllm/transformers_utils/configs/ultravox.py | 120 - .../configs/unlimited_ocr.py | 35 - .../transformers_utils/processors/__init__.py | 32 +- vllm/transformers_utils/processors/bagel.py | 73 - .../processors/cohere_asr.py | 573 -- .../processors/deepseek_vl2.py | 404 -- .../processors/fireredasr2.py | 345 -- vllm/transformers_utils/processors/funasr.py | 429 -- vllm/transformers_utils/processors/glm4v.py | 35 - vllm/transformers_utils/processors/h2ovl.py | 387 -- vllm/transformers_utils/processors/isaac.py | 509 -- .../processors/kimi_audio.py | 105 - .../transformers_utils/processors/kimi_k25.py | 92 - vllm/transformers_utils/processors/kimi_k3.py | 65 - .../processors/mimo_v2_omni.py | 1234 ----- .../transformers_utils/processors/minicpmo.py | 523 -- .../transformers_utils/processors/minicpmv.py | 317 -- .../processors/minimax_m3.py | 660 --- .../processors/moondream3.py | 539 -- .../processors/nemotron_vl.py | 372 -- vllm/transformers_utils/processors/nvlm_d.py | 63 - vllm/transformers_utils/processors/openvla.py | 102 - vllm/transformers_utils/processors/ovis.py | 457 -- vllm/transformers_utils/processors/ovis2_5.py | 476 -- vllm/transformers_utils/processors/pixtral.py | 79 - .../transformers_utils/processors/step3_vl.py | 514 -- vllm/transformers_utils/processors/voxtral.py | 112 - vllm/v1/attention/ops/common.py | 394 -- .../attention/ops/flydsl_kernels/__init__.py | 2 - vllm/v1/attention/ops/pcp.py | 115 - vllm/v1/attention/ops/prefix_prefill.py | 1055 ---- .../attention/ops/triton_decode_attention.py | 817 --- .../ops/triton_unified_attention_diffkv.py | 589 -- vllm/v1/kv_offload/factory.py | 5 - vllm/v1/simple_kv_offload/copy_backend.py | 130 - vllm/v1/simple_kv_offload/disk_backend.py | 343 -- vllm/v1/simple_kv_offload/manager.py | 1458 ----- vllm/v1/simple_kv_offload/metadata.py | 59 - vllm/v1/simple_kv_offload/metrics.py | 284 - vllm/v1/simple_kv_offload/worker.py | 394 -- 353 files changed, 29 insertions(+), 121269 deletions(-) delete mode 100644 tests/distributed/test_packed_tensor.py delete mode 100644 tests/distributed/test_sharded_rdt_trainer.py delete mode 100644 tests/distributed/test_weight_transfer.py delete mode 100644 tests/distributed/test_weight_transfer_nccl_uid.py delete mode 100644 tests/entrypoints/rl/weight_transfer_metrics/test_across_api_servers_metrics.py delete mode 100644 tests/kernels/attention/test_pack_unpack_triton.py delete mode 100644 tests/models/transformers/fusers/test_linear.py delete mode 100644 tests/models/transformers/fusers/test_mla.py delete mode 100644 tests/models/transformers/fusers/test_moe.py delete mode 100644 tests/models/transformers/fusers/test_rms_norm.py delete mode 100644 tests/models/transformers/test_dynamic_rope.py delete mode 100644 tests/models/transformers/test_layer_registry.py delete mode 100644 tests/parser/cohere/test_citations.py delete mode 100644 tests/parser/cohere/test_structural_tags.py delete mode 100644 tests/parser/engine/test_delegating_replay.py delete mode 100644 tests/parser/engine/test_replay.py delete mode 100644 tests/parser/engine/trace_builder.py delete mode 100644 tests/platforms/test_cpu.py delete mode 100644 tests/reasoning/test_base_thinking_reasoning_parser.py delete mode 100644 tests/test_pcp_dp.py delete mode 100644 tests/transformers_utils/processors/test_pixtral.py delete mode 100644 tests/transformers_utils/processors/test_voxtral.py delete mode 100644 tests/transformers_utils/test_bailing_moe_v3_vl_config.py delete mode 100644 tests/transformers_utils/test_dspark_mla_config.py delete mode 100644 tests/v1/kv_connector/unit/test_bidirectional_kv_transfer.py delete mode 100644 tests/v1/kv_connector/unit/test_hf3fs_client.py delete mode 100644 tests/v1/kv_connector/unit/test_hf3fs_connector.py delete mode 100644 tests/v1/kv_connector/unit/test_hf3fs_metadata_server.py delete mode 100644 tests/v1/kv_connector/unit/test_hisparse_stats.py delete mode 100644 tests/v1/kv_connector/unit/test_moriio_hma_scheduler.py delete mode 100644 tests/v1/kv_connector/unit/test_moriio_proxy_routing.py delete mode 100644 tests/v1/kv_connector/unit/test_moriio_routing.py delete mode 100644 tests/v1/kv_connector/unit/test_moriio_routing_fairness.py delete mode 100644 tests/v1/kv_connector/unit/test_moriio_tp_ack.py delete mode 100644 tests/v1/kv_connector/unit/test_moriio_unmap.py delete mode 100644 tests/v1/kv_connector/unit/test_moriio_write_done_routing.py delete mode 100644 tests/v1/kv_connector/unit/test_nixl_connector.py delete mode 100644 tests/v1/kv_connector/unit/test_nixl_heartbeat.py delete mode 100644 tests/v1/kv_connector/unit/test_nixl_push_connector.py delete mode 100644 tests/v1/kv_connector/unit/test_remote_prefill_lifecycle.py delete mode 100644 tests/v1/kv_connector/unit/test_simple_cpu_offload_connector.py delete mode 100644 tests/v1/kv_connector/unit/test_simple_cpu_offload_metrics.py delete mode 100644 tests/v1/kv_connector/unit/test_tp_mapping.py delete mode 100644 tests/v1/simple_kv_offload/test_scheduler.py delete mode 100644 vllm/distributed/kv_transfer/kv_connector/v1/hf3fs/__init__.py delete mode 100644 vllm/distributed/kv_transfer/kv_connector/v1/hf3fs/hf3fs_client.py delete mode 100644 vllm/distributed/kv_transfer/kv_connector/v1/hf3fs/hf3fs_connector.py delete mode 100644 vllm/distributed/kv_transfer/kv_connector/v1/hf3fs/hf3fs_metadata_server.py delete mode 100644 vllm/distributed/kv_transfer/kv_connector/v1/hf3fs/utils/__init__.py delete mode 100644 vllm/distributed/kv_transfer/kv_connector/v1/hf3fs/utils/common.py delete mode 100644 vllm/distributed/kv_transfer/kv_connector/v1/hf3fs/utils/gather_scatter_helper.py delete mode 100644 vllm/distributed/kv_transfer/kv_connector/v1/hf3fs/utils/hf3fs_mock_client.py delete mode 100644 vllm/distributed/kv_transfer/kv_connector/v1/hf3fs/utils/hf3fs_utils.cpp delete mode 100644 vllm/distributed/kv_transfer/kv_connector/v1/hisparse/__init__.py delete mode 100644 vllm/distributed/kv_transfer/kv_connector/v1/hisparse/connector.py delete mode 100644 vllm/distributed/kv_transfer/kv_connector/v1/hisparse/stats.py delete mode 100644 vllm/distributed/kv_transfer/kv_connector/v1/hisparse/worker.py delete mode 100644 vllm/distributed/kv_transfer/kv_connector/v1/moriio/__init__.py delete mode 100644 vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_common.py delete mode 100644 vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_connector.py delete mode 100644 vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_engine.py delete mode 100644 vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_heartbeat.py delete mode 100644 vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_layout.py delete mode 100644 vllm/distributed/kv_transfer/kv_connector/v1/nixl/__init__.py delete mode 100644 vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_scheduler.py delete mode 100644 vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py delete mode 100644 vllm/distributed/kv_transfer/kv_connector/v1/nixl/connector.py delete mode 100644 vllm/distributed/kv_transfer/kv_connector/v1/nixl/metadata.py delete mode 100644 vllm/distributed/kv_transfer/kv_connector/v1/nixl/pull_scheduler.py delete mode 100644 vllm/distributed/kv_transfer/kv_connector/v1/nixl/pull_worker.py delete mode 100644 vllm/distributed/kv_transfer/kv_connector/v1/nixl/push_scheduler.py delete mode 100644 vllm/distributed/kv_transfer/kv_connector/v1/nixl/push_worker.py delete mode 100644 vllm/distributed/kv_transfer/kv_connector/v1/nixl/scheduler.py delete mode 100644 vllm/distributed/kv_transfer/kv_connector/v1/nixl/stats.py delete mode 100644 vllm/distributed/kv_transfer/kv_connector/v1/nixl/tp_mapping.py delete mode 100644 vllm/distributed/kv_transfer/kv_connector/v1/nixl/utils.py delete mode 100644 vllm/distributed/kv_transfer/kv_connector/v1/nixl/worker.py delete mode 100644 vllm/distributed/kv_transfer/kv_connector/v1/simple_cpu_offload_connector.py delete mode 100644 vllm/distributed/kv_transfer/kv_connector/v1/ssm_conv_transfer_utils.py delete mode 100644 vllm/distributed/weight_transfer/ipc_engine.py delete mode 100644 vllm/distributed/weight_transfer/m2n_common.py delete mode 100644 vllm/distributed/weight_transfer/m2n_engine.py delete mode 100644 vllm/distributed/weight_transfer/m2n_layout.py delete mode 100644 vllm/distributed/weight_transfer/m2n_source.py delete mode 100644 vllm/distributed/weight_transfer/m2n_trainer.py delete mode 100644 vllm/distributed/weight_transfer/modelexpress_engine.py delete mode 100644 vllm/distributed/weight_transfer/nccl_common.py delete mode 100644 vllm/distributed/weight_transfer/nccl_engine.py delete mode 100644 vllm/distributed/weight_transfer/packed_tensor.py delete mode 100644 vllm/distributed/weight_transfer/sharded_rdt_common.py delete mode 100644 vllm/distributed/weight_transfer/sharded_rdt_engine.py delete mode 100644 vllm/distributed/weight_transfer/sharded_rdt_fake.py delete mode 100644 vllm/distributed/weight_transfer/sharded_rdt_trainer.py delete mode 100644 vllm/distributed/weight_transfer/sparse_nccl_engine.py delete mode 100644 vllm/lora/ops/torch_ops/__init__.py delete mode 100644 vllm/lora/ops/torch_ops/lora_ops.py delete mode 100644 vllm/lora/ops/triton_ops/fused_moe_lora_fp8_op.py delete mode 100644 vllm/lora/ops/triton_ops/lora_expand_fp8_op.py delete mode 100644 vllm/lora/ops/triton_ops/lora_shrink_fp8_op.py delete mode 100644 vllm/lora/punica_wrapper/punica_cpu.py delete mode 100644 vllm/model_executor/kernels/attention/dsa/__init__.py delete mode 100644 vllm/model_executor/kernels/attention/dsa/candidate_blocks.py delete mode 100644 vllm/model_executor/kernels/attention/dsa/sparse_mqa_logits.py delete mode 100644 vllm/model_executor/kernels/linear/cute_dsl/gemm_rs_ar.py delete mode 100644 vllm/model_executor/layers/fused_embed_norm.py delete mode 100644 vllm/model_executor/layers/fused_moe/fused_flydsl_moe.py delete mode 100644 vllm/model_executor/layers/fused_moe/prepare_megamoe.py delete mode 100644 vllm/model_executor/layers/fusion/relu2_fp8_quant.py delete mode 100644 vllm/model_executor/layers/indexer_topk.py delete mode 100644 vllm/model_executor/layers/lightning_attn.py delete mode 100644 vllm/model_executor/layers/mamba/short_conv.py delete mode 100644 vllm/model_executor/layers/minimax_rms_norm/__init__.py delete mode 100644 vllm/model_executor/layers/minimax_rms_norm/lamport_workspace.py delete mode 100644 vllm/model_executor/layers/minimax_rms_norm/rms_norm_tp.py delete mode 100644 vllm/model_executor/models/transformers/__init__.py delete mode 100644 vllm/model_executor/models/transformers/base.py delete mode 100644 vllm/model_executor/models/transformers/causal.py delete mode 100644 vllm/model_executor/models/transformers/fuser.py delete mode 100644 vllm/model_executor/models/transformers/fusers/__init__.py delete mode 100644 vllm/model_executor/models/transformers/fusers/attention.py delete mode 100644 vllm/model_executor/models/transformers/fusers/base.py delete mode 100644 vllm/model_executor/models/transformers/fusers/glu.py delete mode 100644 vllm/model_executor/models/transformers/fusers/merged_column.py delete mode 100644 vllm/model_executor/models/transformers/fusers/mla.py delete mode 100644 vllm/model_executor/models/transformers/fusers/moe.py delete mode 100644 vllm/model_executor/models/transformers/fusers/packed_qkv.py delete mode 100644 vllm/model_executor/models/transformers/fusers/qkv.py delete mode 100644 vllm/model_executor/models/transformers/fusers/rms_norm.py delete mode 100644 vllm/model_executor/models/transformers/fx_utils.py delete mode 100644 vllm/model_executor/models/transformers/layers.py delete mode 100644 vllm/model_executor/models/transformers/legacy.py delete mode 100644 vllm/model_executor/models/transformers/moe.py delete mode 100644 vllm/model_executor/models/transformers/multimodal.py delete mode 100644 vllm/model_executor/models/transformers/utils.py delete mode 100644 vllm/models/qwen4_exp/amd/__init__.py delete mode 100644 vllm/models/qwen4_exp/amd/hyperconnection.py delete mode 100644 vllm/models/qwen4_exp/amd/indexer_qsa.py delete mode 100644 vllm/models/qwen4_exp/amd/low_latency_gemm.py delete mode 100644 vllm/models/qwen4_exp/amd/model.py delete mode 100644 vllm/models/qwen4_exp/amd/model_state.py delete mode 100644 vllm/models/qwen4_exp/amd/mtp.py delete mode 100644 vllm/models/qwen4_exp/amd/ops/__init__.py delete mode 100644 vllm/models/qwen4_exp/amd/ops/hc.py delete mode 100644 vllm/models/qwen4_exp/amd/ops/qsa.py delete mode 100644 vllm/models/qwen4_exp/amd/ops/qsa_prepare.py delete mode 100644 vllm/models/qwen4_exp/amd/ops/sequence_parallel.py delete mode 100644 vllm/models/qwen4_exp/amd/ple_layer.py delete mode 100644 vllm/models/qwen4_exp/amd/qsa.py delete mode 100644 vllm/parser/AGENTS.md delete mode 100644 vllm/parser/beam.py delete mode 100644 vllm/parser/cohere_command.py delete mode 100644 vllm/parser/deepseek_v32.py delete mode 100644 vllm/parser/deepseek_v4.py delete mode 100644 vllm/parser/deepseek_v41.py delete mode 100644 vllm/parser/gemma4.py delete mode 100644 vllm/parser/glm47_moe.py delete mode 100644 vllm/parser/granite.py delete mode 100644 vllm/parser/granite_thinking.py delete mode 100644 vllm/parser/inkling.py delete mode 100644 vllm/parser/kimi_k2.py delete mode 100644 vllm/parser/kimi_k3.py delete mode 100644 vllm/parser/ling3.py delete mode 100644 vllm/parser/mimo.py delete mode 100644 vllm/parser/minicpmv.py delete mode 100644 vllm/parser/minimax_m2.py delete mode 100644 vllm/parser/minimax_m3.py delete mode 100644 vllm/parser/nemotron_v3.py delete mode 100644 vllm/parser/plamo3.py delete mode 100644 vllm/parser/seed_oss.py delete mode 100644 vllm/parser/step3p5.py delete mode 100644 vllm/platforms/cpu.py delete mode 100644 vllm/platforms/zen_cpu.py delete mode 100644 vllm/reasoning/AGENTS.md delete mode 100644 vllm/reasoning/basic_parsers.py delete mode 100644 vllm/reasoning/beam_reasoning_parser.py delete mode 100644 vllm/reasoning/cohere_command_reasoning_parser.py delete mode 100644 vllm/reasoning/deepseek_r1_reasoning_parser.py delete mode 100644 vllm/reasoning/deepseek_v3_reasoning_parser.py delete mode 100644 vllm/reasoning/deepseek_v41_engine_reasoning_parser.py delete mode 100644 vllm/reasoning/deepseek_v4_engine_reasoning_parser.py delete mode 100644 vllm/reasoning/ernie45_reasoning_parser.py delete mode 100644 vllm/reasoning/gemma4_engine_reasoning_parser.py delete mode 100644 vllm/reasoning/gemma4_utils.py delete mode 100644 vllm/reasoning/glm47_moe_reasoning_parser.py delete mode 100644 vllm/reasoning/granite_reasoning_parser.py delete mode 100644 vllm/reasoning/granite_thinking_engine_reasoning_parser.py delete mode 100644 vllm/reasoning/hunyuan_a13b_reasoning_parser.py delete mode 100644 vllm/reasoning/hy_v3_reasoning_parser.py delete mode 100644 vllm/reasoning/hy_v4_reasoning_parser.py delete mode 100644 vllm/reasoning/identity_reasoning_parser.py delete mode 100644 vllm/reasoning/inkling_reasoning_parser.py delete mode 100644 vllm/reasoning/k2_horizon_reasoning_parser.py delete mode 100644 vllm/reasoning/kimi_k2_reasoning_parser.py delete mode 100644 vllm/reasoning/kimi_k3_reasoning_parser.py delete mode 100644 vllm/reasoning/ling3_reasoning_parser.py delete mode 100644 vllm/reasoning/mimo_engine_reasoning_parser.py delete mode 100644 vllm/reasoning/minicpmv_reasoning_parser.py delete mode 100644 vllm/reasoning/minimax_m2_reasoning_parser.py delete mode 100644 vllm/reasoning/minimax_m3_reasoning_parser.py delete mode 100644 vllm/reasoning/mistral_reasoning_parser.py delete mode 100644 vllm/reasoning/muse_glimmer_reasoning_parser.py delete mode 100644 vllm/reasoning/nemotron_v3_engine_reasoning_parser.py delete mode 100644 vllm/reasoning/olmo3_reasoning_parser.py delete mode 100644 vllm/reasoning/plamo3_engine_reasoning_parser.py delete mode 100644 vllm/reasoning/poolside_v1_reasoning_parser.py delete mode 100644 vllm/reasoning/response_template_reasoning_parser.py delete mode 100644 vllm/reasoning/seed_oss_engine_reasoning_parser.py delete mode 100644 vllm/reasoning/step3_reasoning_parser.py delete mode 100644 vllm/reasoning/step3p5_engine_reasoning_parser.py delete mode 100644 vllm/third_party/flashmla/__init__.py delete mode 100644 vllm/tokenizers/deepseek_v32.py delete mode 100644 vllm/tokenizers/deepseek_v32_encoding.py delete mode 100644 vllm/tokenizers/deepseek_v4.py delete mode 100644 vllm/tokenizers/deepseek_v41_encoding.py delete mode 100644 vllm/tokenizers/deepseek_v4_encoding.py delete mode 100644 vllm/tool_parsers/AGENTS.md delete mode 100644 vllm/tool_parsers/apertus_tool_parser.py delete mode 100644 vllm/tool_parsers/beam_tool_parser.py delete mode 100644 vllm/tool_parsers/cohere_command_tool_parser.py delete mode 100644 vllm/tool_parsers/deepseekv31_tool_parser.py delete mode 100644 vllm/tool_parsers/deepseekv32_engine_tool_parser.py delete mode 100644 vllm/tool_parsers/deepseekv3_tool_parser.py delete mode 100644 vllm/tool_parsers/deepseekv41_engine_tool_parser.py delete mode 100644 vllm/tool_parsers/deepseekv4_engine_tool_parser.py delete mode 100644 vllm/tool_parsers/dots_tool_parser.py delete mode 100644 vllm/tool_parsers/ernie45_tool_parser.py delete mode 100644 vllm/tool_parsers/functiongemma_tool_parser.py delete mode 100644 vllm/tool_parsers/gemma4_engine_tool_parser.py delete mode 100644 vllm/tool_parsers/gemma4_utils.py delete mode 100644 vllm/tool_parsers/gigachat3_tool_parser.py delete mode 100644 vllm/tool_parsers/glm47_moe_tool_parser.py delete mode 100644 vllm/tool_parsers/granite4_tool_parser.py delete mode 100644 vllm/tool_parsers/granite_20b_fc_tool_parser.py delete mode 100644 vllm/tool_parsers/granite_engine_tool_parser.py delete mode 100644 vllm/tool_parsers/hermes_tool_parser.py delete mode 100644 vllm/tool_parsers/hunyuan_a13b_tool_parser.py delete mode 100644 vllm/tool_parsers/hy_v3_tool_parser.py delete mode 100644 vllm/tool_parsers/hy_v4_tool_parser.py delete mode 100644 vllm/tool_parsers/inkling_tool_parser.py delete mode 100644 vllm/tool_parsers/internlm2_tool_parser.py delete mode 100644 vllm/tool_parsers/jamba_tool_parser.py delete mode 100644 vllm/tool_parsers/k2_horizon_tool_parser.py delete mode 100644 vllm/tool_parsers/kimi_k2_tool_parser.py delete mode 100644 vllm/tool_parsers/kimi_k3_tool_parser.py delete mode 100644 vllm/tool_parsers/lfm2_tool_parser.py delete mode 100644 vllm/tool_parsers/ling3_tool_parser.py delete mode 100644 vllm/tool_parsers/llama4_pythonic_tool_parser.py delete mode 100644 vllm/tool_parsers/llama_tool_parser.py delete mode 100644 vllm/tool_parsers/longcat_tool_parser.py delete mode 100644 vllm/tool_parsers/mimo_tool_parser.py delete mode 100644 vllm/tool_parsers/minicpm5xml_tool_parser.py delete mode 100644 vllm/tool_parsers/minimax_m2_tool_parser.py delete mode 100644 vllm/tool_parsers/minimax_m3_tool_parser.py delete mode 100644 vllm/tool_parsers/mistral_tool_parser.py delete mode 100644 vllm/tool_parsers/muse_glimmer_tool_parser.py delete mode 100644 vllm/tool_parsers/olmo3_tool_parser.py delete mode 100644 vllm/tool_parsers/phi4mini_tool_parser.py delete mode 100644 vllm/tool_parsers/plamo3_engine_tool_parser.py delete mode 100644 vllm/tool_parsers/poolside_v1_tool_parser.py delete mode 100644 vllm/tool_parsers/pythonic_tool_parser.py delete mode 100644 vllm/tool_parsers/response_template_tool_parser.py delete mode 100644 vllm/tool_parsers/seed_oss_engine_tool_parser.py delete mode 100644 vllm/tool_parsers/step3_tool_parser.py delete mode 100644 vllm/tool_parsers/step3p5_tool_parser.py delete mode 100644 vllm/tool_parsers/xlam_tool_parser.py delete mode 100644 vllm/transformers_utils/configs/bailing_moe_v3_vl.py delete mode 100644 vllm/transformers_utils/configs/chatglm.py delete mode 100644 vllm/transformers_utils/configs/colmodernvbert.py delete mode 100644 vllm/transformers_utils/configs/colpali.py delete mode 100644 vllm/transformers_utils/configs/cosmos3_edge.py delete mode 100644 vllm/transformers_utils/configs/deepseek_v41.py delete mode 100644 vllm/transformers_utils/configs/deepseek_vl2.py delete mode 100644 vllm/transformers_utils/configs/dots3_note.py delete mode 100644 vllm/transformers_utils/configs/falcon.py delete mode 100644 vllm/transformers_utils/configs/funaudiochat.py delete mode 100644 vllm/transformers_utils/configs/hy_v4.py delete mode 100644 vllm/transformers_utils/configs/isaac.py delete mode 100644 vllm/transformers_utils/configs/k3_dspark.py delete mode 100644 vllm/transformers_utils/configs/kimi_k3.py delete mode 100644 vllm/transformers_utils/configs/kimi_linear.py delete mode 100644 vllm/transformers_utils/configs/kimi_vl.py delete mode 100644 vllm/transformers_utils/configs/midashenglm.py delete mode 100644 vllm/transformers_utils/configs/minimax_m3.py delete mode 100644 vllm/transformers_utils/configs/moondream3.py delete mode 100644 vllm/transformers_utils/configs/moonvit.py delete mode 100644 vllm/transformers_utils/configs/moss_transcribe_diarize.py delete mode 100644 vllm/transformers_utils/configs/muse_glimmer.py delete mode 100644 vllm/transformers_utils/configs/openvla.py delete mode 100644 vllm/transformers_utils/configs/ovis.py delete mode 100644 vllm/transformers_utils/configs/qianfan_ocr.py delete mode 100644 vllm/transformers_utils/configs/qwen3_asr.py delete mode 100644 vllm/transformers_utils/configs/radio.py delete mode 100644 vllm/transformers_utils/configs/step3_vl.py delete mode 100644 vllm/transformers_utils/configs/step3p5.py delete mode 100644 vllm/transformers_utils/configs/ultravox.py delete mode 100644 vllm/transformers_utils/configs/unlimited_ocr.py delete mode 100644 vllm/transformers_utils/processors/bagel.py delete mode 100644 vllm/transformers_utils/processors/cohere_asr.py delete mode 100644 vllm/transformers_utils/processors/deepseek_vl2.py delete mode 100644 vllm/transformers_utils/processors/fireredasr2.py delete mode 100644 vllm/transformers_utils/processors/funasr.py delete mode 100644 vllm/transformers_utils/processors/glm4v.py delete mode 100644 vllm/transformers_utils/processors/h2ovl.py delete mode 100644 vllm/transformers_utils/processors/isaac.py delete mode 100644 vllm/transformers_utils/processors/kimi_audio.py delete mode 100644 vllm/transformers_utils/processors/kimi_k25.py delete mode 100644 vllm/transformers_utils/processors/kimi_k3.py delete mode 100644 vllm/transformers_utils/processors/mimo_v2_omni.py delete mode 100644 vllm/transformers_utils/processors/minicpmo.py delete mode 100644 vllm/transformers_utils/processors/minicpmv.py delete mode 100644 vllm/transformers_utils/processors/minimax_m3.py delete mode 100644 vllm/transformers_utils/processors/moondream3.py delete mode 100644 vllm/transformers_utils/processors/nemotron_vl.py delete mode 100644 vllm/transformers_utils/processors/nvlm_d.py delete mode 100644 vllm/transformers_utils/processors/openvla.py delete mode 100644 vllm/transformers_utils/processors/ovis.py delete mode 100644 vllm/transformers_utils/processors/ovis2_5.py delete mode 100644 vllm/transformers_utils/processors/pixtral.py delete mode 100644 vllm/transformers_utils/processors/step3_vl.py delete mode 100644 vllm/transformers_utils/processors/voxtral.py delete mode 100644 vllm/v1/attention/ops/common.py delete mode 100644 vllm/v1/attention/ops/flydsl_kernels/__init__.py delete mode 100644 vllm/v1/attention/ops/pcp.py delete mode 100644 vllm/v1/attention/ops/prefix_prefill.py delete mode 100644 vllm/v1/attention/ops/triton_decode_attention.py delete mode 100644 vllm/v1/attention/ops/triton_unified_attention_diffkv.py delete mode 100644 vllm/v1/simple_kv_offload/copy_backend.py delete mode 100644 vllm/v1/simple_kv_offload/disk_backend.py delete mode 100644 vllm/v1/simple_kv_offload/manager.py delete mode 100644 vllm/v1/simple_kv_offload/metadata.py delete mode 100644 vllm/v1/simple_kv_offload/metrics.py delete mode 100644 vllm/v1/simple_kv_offload/worker.py diff --git a/tests/distributed/test_packed_tensor.py b/tests/distributed/test_packed_tensor.py deleted file mode 100644 index 63af7f4b56..0000000000 --- a/tests/distributed/test_packed_tensor.py +++ /dev/null @@ -1,775 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Tests for packed tensor broadcasting functionality. - -Unit tests for packed_nccl_broadcast_producer and packed_nccl_broadcast_consumer. -These utilities enable efficient batched tensor transfer over NCCL. -""" - -import pytest -import torch - -from vllm.distributed.weight_transfer.packed_tensor import ( - pack_tensors, - packed_ipc_consumer, - packed_ipc_producer, - packed_nccl_broadcast_consumer, - packed_nccl_broadcast_producer, - unpack_tensor, -) - - -class MockCommunicationGroup: - """Mock communication group for testing producer broadcast operations.""" - - def __init__(self): - self.broadcasted_tensors: list[torch.Tensor] = [] - self.broadcast_count = 0 - self.device = torch.device("cuda:0") - - def broadcast(self, tensor, src): - """Mock broadcast that stores the tensor for later verification.""" - self.broadcasted_tensors.append(tensor.clone()) - self.broadcast_count += 1 - - -class MockConsumerCommunicationGroup: - """Mock communication group for consumer that returns pre-stored tensors.""" - - def __init__(self, tensors_to_return: list[torch.Tensor]): - self.tensors_to_return = tensors_to_return - self.current_index = 0 - self.device = torch.device("cuda:0") - - def broadcast(self, tensor, src): - """Mock broadcast that fills the tensor with pre-stored data.""" - if self.current_index < len(self.tensors_to_return): - tensor.copy_(self.tensors_to_return[self.current_index]) - self.current_index += 1 - - -def create_mock_model_params( - num_layers: int = 3, - dtype: torch.dtype = torch.float32, -) -> list[tuple[str, torch.Tensor]]: - """Create mock model parameters for testing.""" - params = [] - for i in range(num_layers): - params.append((f"layer{i}.weight", torch.randn(10, 20, dtype=dtype))) - params.append((f"layer{i}.bias", torch.randn(10, dtype=dtype))) - return params - - -def create_state_dict_info( - params: list[tuple[str, torch.Tensor]], -) -> dict[str, tuple[tuple[int, ...], torch.dtype]]: - """Create state dict info (name -> (shape, dtype)) from params.""" - return {name: (tuple(tensor.shape), tensor.dtype) for name, tensor in params} - - -# --- Unit Tests: packed_nccl_broadcast_producer --- - - -@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available") -class TestPackedBroadcastProducer: - """Test packed_nccl_broadcast_producer function.""" - - def test_producer_empty_iterator(self): - """Test producer handles empty iterator gracefully.""" - mock_group = MockCommunicationGroup() - - packed_nccl_broadcast_producer( - iterator=iter([]), - group=mock_group, - src=0, - post_iter_func=lambda x: x[1], - buffer_size_bytes=1000, - ) - - # No broadcasts for empty iterator - assert mock_group.broadcast_count == 0 - - -# --- Integration Tests: Producer-Consumer Roundtrip --- - - -@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available") -class TestPackedBroadcastRoundtrip: - """Test producer-consumer roundtrip behavior.""" - - @pytest.mark.parametrize("dtype", [torch.float32, torch.float16, torch.bfloat16]) - def test_roundtrip_different_dtypes(self, dtype): - """Test roundtrip with different data types.""" - params = create_mock_model_params(num_layers=2, dtype=dtype) - params_cuda = [(name, tensor.cuda()) for name, tensor in params] - - buffer_size = 1000 - producer_group = MockCommunicationGroup() - - packed_nccl_broadcast_producer( - iterator=iter(params_cuda), - group=producer_group, - src=0, - post_iter_func=lambda x: x[1], - buffer_size_bytes=buffer_size, - ) - - consumer_group = MockConsumerCommunicationGroup( - producer_group.broadcasted_tensors - ) - - state_dict_info = create_state_dict_info(params_cuda) - unpacked_tensors = {} - - def post_unpack_func(tensor_list): - for name, tensor in tensor_list: - unpacked_tensors[name] = tensor.clone() - - packed_nccl_broadcast_consumer( - iterator=iter(state_dict_info.items()), - group=consumer_group, - src=0, - post_unpack_func=post_unpack_func, - buffer_size_bytes=buffer_size, - ) - - # Verify roundtrip preserves data - for name, original_tensor in params_cuda: - assert name in unpacked_tensors - unpacked = unpacked_tensors[name] - assert unpacked.dtype == dtype - assert torch.allclose(unpacked, original_tensor, rtol=1e-4, atol=1e-6) - - def test_roundtrip_mixed_dtypes(self): - """Test roundtrip with mixed data types.""" - # Create params with mixed dtypes - params = [ - ("layer1.weight", torch.randn(10, 20, dtype=torch.float32).cuda()), - ("layer1.bias", torch.randn(10, dtype=torch.float16).cuda()), - ("layer2.weight", torch.randn(20, 30, dtype=torch.bfloat16).cuda()), - ] - - buffer_size = 500 - producer_group = MockCommunicationGroup() - - packed_nccl_broadcast_producer( - iterator=iter(params), - group=producer_group, - src=0, - post_iter_func=lambda x: x[1], - buffer_size_bytes=buffer_size, - ) - - consumer_group = MockConsumerCommunicationGroup( - producer_group.broadcasted_tensors - ) - - state_dict_info = create_state_dict_info(params) - unpacked_tensors = {} - - def post_unpack_func(tensor_list): - for name, tensor in tensor_list: - unpacked_tensors[name] = tensor.clone() - - packed_nccl_broadcast_consumer( - iterator=iter(state_dict_info.items()), - group=consumer_group, - src=0, - post_unpack_func=post_unpack_func, - buffer_size_bytes=buffer_size, - ) - - # Verify all params roundtrip correctly with correct dtypes - for name, original_tensor in params: - assert name in unpacked_tensors - unpacked = unpacked_tensors[name] - assert unpacked.shape == original_tensor.shape - assert unpacked.dtype == original_tensor.dtype - assert torch.allclose(unpacked, original_tensor, rtol=1e-4, atol=1e-6) - - @pytest.mark.parametrize("target_size", [100, 100000]) - def test_roundtrip_different_batch_sizes(self, target_size): - """Test roundtrip with different target batch sizes.""" - params = create_mock_model_params(num_layers=5) - params_cuda = [(name, tensor.cuda()) for name, tensor in params] - - producer_group = MockCommunicationGroup() - - packed_nccl_broadcast_producer( - iterator=iter(params_cuda), - group=producer_group, - src=0, - post_iter_func=lambda x: x[1], - buffer_size_bytes=target_size, - ) - - consumer_group = MockConsumerCommunicationGroup( - producer_group.broadcasted_tensors - ) - - state_dict_info = create_state_dict_info(params_cuda) - unpacked_tensors = {} - - def post_unpack_func(tensor_list): - for name, tensor in tensor_list: - unpacked_tensors[name] = tensor.clone() - - packed_nccl_broadcast_consumer( - iterator=iter(state_dict_info.items()), - group=consumer_group, - src=0, - post_unpack_func=post_unpack_func, - buffer_size_bytes=target_size, - ) - - # Verify all params roundtrip correctly - assert len(unpacked_tensors) == len(params) - for name, original_tensor in params_cuda: - assert name in unpacked_tensors - assert torch.allclose( - unpacked_tensors[name], original_tensor, rtol=1e-5, atol=1e-7 - ) - - def test_roundtrip_non_contiguous_tensors(self): - """Test roundtrip with non-contiguous tensors from the trainer.""" - # Create non-contiguous tensors (simulating trainer outputs) - # Transposed tensors are non-contiguous - weight1 = torch.randn(20, 10, dtype=torch.float32).cuda().T - # Sliced tensors with step are non-contiguous - weight2 = torch.randn(40, 30, dtype=torch.float16).cuda()[::2, ::2] - # Permuted tensors are non-contiguous - weight3 = torch.randn(5, 10, 15, dtype=torch.bfloat16).cuda().permute(2, 0, 1) - - params = [ - ("layer1.weight", weight1), - ("layer2.weight", weight2), - ("layer3.weight", weight3), - ] - - # Verify tensors are indeed non-contiguous - for name, tensor in params: - assert not tensor.is_contiguous(), f"{name} should be non-contiguous" - - buffer_size = 500 - producer_group = MockCommunicationGroup() - - packed_nccl_broadcast_producer( - iterator=iter(params), - group=producer_group, - src=0, - post_iter_func=lambda x: x[1], - buffer_size_bytes=buffer_size, - ) - - consumer_group = MockConsumerCommunicationGroup( - producer_group.broadcasted_tensors - ) - - state_dict_info = create_state_dict_info(params) - unpacked_tensors = {} - - def post_unpack_func(tensor_list): - for name, tensor in tensor_list: - unpacked_tensors[name] = tensor.clone() - - packed_nccl_broadcast_consumer( - iterator=iter(state_dict_info.items()), - group=consumer_group, - src=0, - post_unpack_func=post_unpack_func, - buffer_size_bytes=buffer_size, - ) - - # Verify all non-contiguous params roundtrip correctly - for name, original_tensor in params: - assert name in unpacked_tensors - unpacked = unpacked_tensors[name] - assert unpacked.shape == original_tensor.shape - assert unpacked.dtype == original_tensor.dtype - assert torch.allclose(unpacked, original_tensor, rtol=1e-4, atol=1e-6) - - -# --- Unit Tests: unpack_tensor --- - - -@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available") -class TestUnpackTensor: - """Test the shared unpack_tensor function.""" - - def test_unpack_produces_independent_copies(self): - """Verify unpacked tensors don't share memory with packed buffer.""" - original = torch.randn(10, dtype=torch.float32).cuda() - packed = original.contiguous().view(torch.uint8).view(-1) - - result = unpack_tensor( - packed, - names=["w"], - shapes=[[10]], - dtypes=[torch.float32], - tensor_sizes=[packed.numel()], - ) - - # Mutate the packed buffer - packed.zero_() - - # Unpacked tensor should be unaffected - assert torch.allclose(result[0][1], original) - - -# --- Unit Tests: pack_tensors --- - - -@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available") -class TestPackTensors: - """Test the shared pack_tensors function.""" - - def test_pack_basic(self): - """Test packing a few tensors into one buffer.""" - params = [ - ("w1", torch.randn(10, 20, dtype=torch.float32).cuda()), - ("w2", torch.randn(5, dtype=torch.float16).cuda()), - ] - - chunk = pack_tensors( - iterator=iter(params), - post_iter_func=lambda x: x[1], - buffer_size_bytes=10_000_000, - ) - - assert chunk is not None - assert len(chunk.names) == 2 - assert chunk.names == ["w1", "w2"] - assert chunk.shapes == [[10, 20], [5]] - assert chunk.dtypes == [torch.float32, torch.float16] - assert chunk.packed_tensor.dtype == torch.uint8 - - def test_pack_respects_buffer_limit(self): - """Test that packing stops when buffer_size_bytes is exceeded.""" - params = [ - (f"w{i}", torch.randn(100, 100, dtype=torch.float32).cuda()) - for i in range(10) - ] - - chunk = pack_tensors( - iterator=iter(params), - post_iter_func=lambda x: x[1], - buffer_size_bytes=50_000, - ) - - assert chunk is not None - assert len(chunk.names) < 10 - - def test_pack_empty_iterator(self): - """Test that an empty iterator returns None.""" - chunk = pack_tensors( - iterator=iter([]), - post_iter_func=lambda x: x[1], - buffer_size_bytes=1000, - ) - assert chunk is None - - def test_pack_single_tensor_larger_than_buffer_warns(self): - """Test that a tensor exceeding buffer_size_bytes emits a warning.""" - big = torch.randn(1000, 1000, dtype=torch.float32).cuda() - params = [("big", big)] - - import warnings - - with warnings.catch_warnings(record=True) as w: - warnings.simplefilter("always") - chunk = pack_tensors( - iterator=iter(params), - post_iter_func=lambda x: x[1], - buffer_size_bytes=100, - ) - assert chunk is not None - assert len(chunk.names) == 1 - assert any("exceeds buffer_size_bytes" in str(wi.message) for wi in w) - - def test_pack_unpack_roundtrip(self): - """Test pack then unpack produces identical tensors.""" - params = [ - ("a", torch.randn(8, 16, dtype=torch.float32).cuda()), - ("b", torch.randn(4, dtype=torch.float16).cuda()), - ("c", torch.randn(3, 5, 7, dtype=torch.bfloat16).cuda()), - ] - - chunk = pack_tensors( - iterator=iter(params), - post_iter_func=lambda x: x[1], - buffer_size_bytes=10_000_000, - ) - - assert chunk is not None - result = unpack_tensor( - chunk.packed_tensor, - chunk.names, - chunk.shapes, - chunk.dtypes, - chunk.tensor_sizes, - ) - - assert len(result) == len(params) - for (orig_name, orig_tensor), (res_name, res_tensor) in zip(params, result): - assert orig_name == res_name - assert res_tensor.shape == orig_tensor.shape - assert res_tensor.dtype == orig_tensor.dtype - assert torch.allclose(res_tensor, orig_tensor, rtol=1e-4, atol=1e-6) - - def test_pack_multiple_chunks(self): - """Test consuming an iterator across multiple pack_tensors calls.""" - params = [ - (f"w{i}", torch.randn(50, 50, dtype=torch.float32).cuda()) for i in range(6) - ] - it = iter(params) - - all_names = [] - chunks = [] - while True: - chunk = pack_tensors(it, lambda x: x[1], buffer_size_bytes=12_000) - if chunk is None: - break - chunks.append(chunk) - all_names.extend(chunk.names) - - assert len(chunks) > 1 - assert all_names == [f"w{i}" for i in range(6)] - - -# --- Unit Tests: packed_ipc_producer --- - - -@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available") -class TestPackedIpcProducer: - """Test the packed_ipc_producer generator.""" - - def test_producer_yields_chunks(self): - """Test that the producer yields PackedIpcChunk objects.""" - params = [ - (f"w{i}", torch.randn(50, 50, dtype=torch.float32).cuda()) for i in range(6) - ] - - chunks = list( - packed_ipc_producer( - iterator=iter(params), - gpu_uuid="test-uuid", - post_iter_func=lambda x: x[1], - buffer_size_bytes=12_000, - ) - ) - - assert len(chunks) > 1 - - def test_producer_ipc_handle_has_uuid(self): - """Test that each chunk's ipc_handle is keyed by the given UUID.""" - params = [("w", torch.randn(10, dtype=torch.float32).cuda())] - - chunks = list( - packed_ipc_producer( - iterator=iter(params), - gpu_uuid="my-gpu-uuid", - post_iter_func=lambda x: x[1], - buffer_size_bytes=10_000_000, - ) - ) - - assert "my-gpu-uuid" in chunks[0].ipc_handle - - def test_producer_dtype_names_are_strings(self): - """Test that dtype_names are string representations.""" - params = [ - ("a", torch.randn(10, dtype=torch.float32).cuda()), - ("b", torch.randn(10, dtype=torch.float16).cuda()), - ] - - chunks = list( - packed_ipc_producer( - iterator=iter(params), - gpu_uuid="uuid", - post_iter_func=lambda x: x[1], - buffer_size_bytes=10_000_000, - ) - ) - - assert chunks[0].dtype_names == ["float32", "float16"] - - def test_producer_empty_iterator(self): - """Test producer with empty iterator yields nothing.""" - chunks = list( - packed_ipc_producer( - iterator=iter([]), - gpu_uuid="uuid", - post_iter_func=lambda x: x[1], - buffer_size_bytes=1000, - ) - ) - assert len(chunks) == 0 - - -# --- Integration Tests: IPC Producer-Consumer Roundtrip --- - - -def _ipc_consumer_worker(cmd_q, ack_q, result_q, done_event, device_index): - """Worker that consumes chunks streamed one at a time from the parent. - - CUDA IPC requires the consumer to be in a separate process from the - producer. The producer reuses a single IPC buffer between chunks, so - the parent must wait for our ack (sent after we copy the chunk to - CPU) before advancing the producer. - """ - try: - torch.accelerator.set_device_index(device_index) - all_results = [] - while True: - cd = cmd_q.get() - if cd is None: - break - result = packed_ipc_consumer( - ipc_handle=cd["ipc_handle"], - names=cd["names"], - shapes=cd["shapes"], - dtype_names=cd["dtype_names"], - tensor_sizes=cd["tensor_sizes"], - device_index=device_index, - ) - # .cpu() forces a GPU→CPU copy off the shared IPC buffer, so - # the producer is free to overwrite it once we ack. - all_results.extend([(name, tensor.cpu()) for name, tensor in result]) - del result - ack_q.put("ack") - result_q.put(("ok", all_results)) - except Exception as e: - result_q.put(("error", str(e))) - # Keep the process alive until the parent has finished reading from - # the result queue — torch serializes CPU tensors via fd sharing, - # which requires this process's resource-sharer server to be running. - done_event.wait(timeout=60) - - -@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available") -class TestPackedIpcRoundtrip: - """Test IPC producer-consumer roundtrip using real CUDA IPC. - - These tests spawn a child process for the consumer because - rebuild_cuda_tensor requires a separate process from the one that - called reduce_tensor. - """ - - def _get_gpu_uuid(self) -> str: - device_index = torch.cuda.current_device() - props = torch.cuda.get_device_properties(device_index) - return str(props.uuid) - - def _run_roundtrip(self, chunk_iter, device_index, timeout=30): - """Stream chunks through a child consumer one at a time. - - ``packed_ipc_producer`` reuses a single IPC buffer for every - chunk, so the producer must not be advanced until the consumer - has finished reading the current chunk. We enforce that with an - ack queue: the consumer puts ``"ack"`` after it has copied the - chunk to CPU, and only then do we pull the next chunk from the - generator. - - Returns ``(num_chunks, results)``. - """ - import multiprocessing as mp - - ctx = mp.get_context("spawn") - cmd_q = ctx.Queue() - ack_q = ctx.Queue() - result_q = ctx.Queue() - done_event = ctx.Event() - proc = ctx.Process( - target=_ipc_consumer_worker, - args=(cmd_q, ack_q, result_q, done_event, device_index), - ) - proc.start() - - num_chunks = 0 - try: - for chunk in chunk_iter: - cmd_q.put( - { - "ipc_handle": chunk.ipc_handle, - "names": chunk.names, - "shapes": chunk.shapes, - "dtype_names": chunk.dtype_names, - "tensor_sizes": chunk.tensor_sizes, - } - ) - if ack_q.get(timeout=timeout) != "ack": - raise RuntimeError("Consumer did not ack chunk") - num_chunks += 1 - cmd_q.put(None) - status, payload = result_q.get(timeout=timeout) - finally: - done_event.set() - proc.join(timeout=10) - if proc.is_alive(): - proc.kill() - - if status == "error": - raise RuntimeError(f"Consumer process failed: {payload}") - # Reclaim IPC-shared memory now that the child has released it - torch.cuda.ipc_collect() - return num_chunks, payload - - def test_roundtrip_basic(self): - """Test basic IPC producer -> consumer roundtrip.""" - params = [ - ("w1", torch.randn(10, 20, dtype=torch.float32).cuda()), - ("scale", torch.tensor(1.5, dtype=torch.float32).cuda()), - ("w2", torch.randn(5, dtype=torch.float16).cuda()), - ] - gpu_uuid = self._get_gpu_uuid() - device_index = torch.cuda.current_device() - - num_chunks, result = self._run_roundtrip( - packed_ipc_producer( - iterator=iter(params), - gpu_uuid=gpu_uuid, - post_iter_func=lambda x: x[1], - buffer_size_bytes=10_000_000, - ), - device_index, - ) - - assert num_chunks == 1 - assert len(result) == len(params) - for (orig_name, orig_tensor), (res_name, res_tensor) in zip(params, result): - assert orig_name == res_name - assert res_tensor.shape == orig_tensor.shape - assert res_tensor.dtype == orig_tensor.dtype - assert torch.allclose(res_tensor, orig_tensor.cpu(), rtol=1e-4, atol=1e-6) - - @pytest.mark.parametrize("dtype", [torch.float32, torch.float16, torch.bfloat16]) - def test_roundtrip_dtypes(self, dtype): - """Test IPC roundtrip with different dtypes.""" - params = create_mock_model_params(num_layers=2, dtype=dtype) - params_cuda = [(n, t.cuda()) for n, t in params] - gpu_uuid = self._get_gpu_uuid() - device_index = torch.cuda.current_device() - - _, result = self._run_roundtrip( - packed_ipc_producer( - iterator=iter(params_cuda), - gpu_uuid=gpu_uuid, - post_iter_func=lambda x: x[1], - buffer_size_bytes=10_000_000, - ), - device_index, - ) - - assert len(result) == len(params_cuda) - for (orig_name, orig_tensor), (res_name, res_tensor) in zip( - params_cuda, result - ): - assert orig_name == res_name - assert res_tensor.dtype == dtype - assert torch.allclose(res_tensor, orig_tensor.cpu(), rtol=1e-4, atol=1e-6) - - def test_roundtrip_multiple_chunks(self): - """Test IPC roundtrip across multiple chunks.""" - params = [ - (f"layer{i}.weight", torch.randn(100, 100, dtype=torch.float32).cuda()) - for i in range(8) - ] - gpu_uuid = self._get_gpu_uuid() - device_index = torch.cuda.current_device() - - num_chunks, result = self._run_roundtrip( - packed_ipc_producer( - iterator=iter(params), - gpu_uuid=gpu_uuid, - post_iter_func=lambda x: x[1], - buffer_size_bytes=50_000, - ), - device_index, - ) - - assert num_chunks > 1 - assert len(result) == len(params) - for (orig_name, orig_tensor), (res_name, res_tensor) in zip(params, result): - assert orig_name == res_name - assert torch.allclose(res_tensor, orig_tensor.cpu(), rtol=1e-5, atol=1e-7) - - def test_roundtrip_non_contiguous(self): - """Test IPC roundtrip with non-contiguous tensors.""" - params = [ - ("transposed", torch.randn(20, 10, dtype=torch.float32).cuda().T), - ("sliced", torch.randn(40, 30, dtype=torch.float16).cuda()[::2, ::2]), - ] - gpu_uuid = self._get_gpu_uuid() - device_index = torch.cuda.current_device() - - for _, t in params: - assert not t.is_contiguous() - - _, result = self._run_roundtrip( - packed_ipc_producer( - iterator=iter(params), - gpu_uuid=gpu_uuid, - post_iter_func=lambda x: x[1], - buffer_size_bytes=10_000_000, - ), - device_index, - ) - - for (orig_name, orig_tensor), (res_name, res_tensor) in zip(params, result): - assert orig_name == res_name - assert res_tensor.shape == orig_tensor.shape - assert res_tensor.dtype == orig_tensor.dtype - assert torch.allclose(res_tensor, orig_tensor.cpu(), rtol=1e-4, atol=1e-6) - - def test_consumer_wrong_uuid_raises(self): - """Test that consumer raises ValueError for unknown GPU UUID.""" - params = [("w", torch.randn(10, dtype=torch.float32).cuda())] - gpu_uuid = self._get_gpu_uuid() - - chunks = list( - packed_ipc_producer( - iterator=iter(params), - gpu_uuid=gpu_uuid, - post_iter_func=lambda x: x[1], - buffer_size_bytes=10_000_000, - ) - ) - - c = chunks[0] - fake_handle = {"fake-uuid-12345": c.ipc_handle[gpu_uuid]} - - with pytest.raises(ValueError, match="IPC handle not found"): - packed_ipc_consumer( - ipc_handle=fake_handle, - names=c.names, - shapes=c.shapes, - dtype_names=c.dtype_names, - tensor_sizes=c.tensor_sizes, - device_index=torch.cuda.current_device(), - ) - - -@pytest.mark.parametrize("dtype", [torch.float32, torch.float16, torch.bfloat16]) -def test_pack_unpack_scalar(dtype): - """Preserve scalar shape, dtype and value alongside a matrix on CPU.""" - params = [ - ("weight", torch.randn(2, 3, dtype=dtype)), - ("scale", torch.tensor(1.5, dtype=dtype)), - ] - - chunk = pack_tensors(iter(params), lambda item: item[1], buffer_size_bytes=1024) - assert chunk is not None - - result = unpack_tensor( - chunk.packed_tensor, - chunk.names, - chunk.shapes, - chunk.dtypes, - chunk.tensor_sizes, - ) - - for (expected_name, expected), (name, actual) in zip(params, result, strict=True): - assert name == expected_name - assert actual.shape == expected.shape - assert actual.dtype == expected.dtype - torch.testing.assert_close(actual, expected, rtol=0, atol=0) diff --git a/tests/distributed/test_sharded_rdt_trainer.py b/tests/distributed/test_sharded_rdt_trainer.py deleted file mode 100644 index 9339e503c4..0000000000 --- a/tests/distributed/test_sharded_rdt_trainer.py +++ /dev/null @@ -1,628 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Tests for the sharded-RDT trainer-side engine. - -The trainer engine's own surface: init info and factory registration, the -send_weights round trip against an in-process fake producer server, the -ownership table it resolves and ships to the workers, and the producer's packed -destination-view cache. - -The plan/replay half (including `RdtRouter`, which is consumer-side) lives in -test_sharded_rdt_plan.py; the producer server's protocol in -test_sharded_rdt_producer.py. -""" - -import itertools -from dataclasses import asdict - -import pytest -import torch - -from tests.distributed.test_weight_transfer import RecordingClient -from vllm.distributed.weight_transfer import ( - ModuleSource, - ParamMeta, - WeightSource, - WeightTransferTrainerFactory, -) -from vllm.distributed.weight_transfer.base import layerwise_groups -from vllm.distributed.weight_transfer.sharded_rdt_trainer import ( - ShardedRDTTrainerInitInfo, - ShardedRDTTrainerWeightTransferEngine, -) - - -class _ListSource(WeightSource): - """A WeightSource over an explicit ordered (name, cpu-tensor) list, so the - sharded-RDT group/order logic can be tested without a real model.""" - - def __init__(self, pairs): - self._pairs = list(pairs) - - def metadata(self): - return [ParamMeta(n, t.dtype, tuple(t.shape)) for n, t in self._pairs] - - def __iter__(self): - return iter(self._pairs) - - -class _FakeProducerServer: - """In-process stand-in for the _RDTProducerServer Ray actor. Records the - engine->server call sequence and, by default, frees each group as soon as - it is published (simulating the consumers' free_group barrier) so the - gather loop's credit gate never parks. Mirrors the real server's - per-group barrier: publish/free are keyed by GROUP INDEX, signals count - to the ``begin_sync`` live total, publish_group returns nothing, and freed - groups flow back only through wait_freed / end_sync (see - test_sharded_rdt_producer.TestFakeServerAgreesWithTheRealOne).""" - - def __init__(self, auto_free=True): - self.order: list[str] = [] - self.published: list[int] = [] - self.live_count = 1 - self._inflight_groups: list[int] = [] - self.auto_free = auto_free - self.free_counts: dict[int, int] = {} - self._pending_freed: list[int] = [] - - # Alias for tests that inspect in-flight groups. - @property - def inflight(self): - return self._inflight_groups - - def begin_sync(self, live_count, live_consumer_ids=None): - self.order.append("begin") - self.live_count = max(1, int(live_count)) - self.live_ids = live_consumer_ids - - def publish_group(self, group_idx, entries): - self.order.append("publish") - self.published.append(group_idx) - self._inflight_groups.append(group_idx) - if self.auto_free or self.free_counts.get(group_idx, 0) >= self.live_count: - self._inflight_groups.remove(group_idx) - self._pending_freed.append(group_idx) - - def free_group(self, group_idx): - """Consumer back-edge; may arrive before the group's publish.""" - self.free_counts[group_idx] = self.free_counts.get(group_idx, 0) + 1 - if ( - group_idx in self._inflight_groups - and self.free_counts[group_idx] >= self.live_count - ): - self._inflight_groups.remove(group_idx) - self._pending_freed.append(group_idx) - - def wait_freed(self): - """The engine's credit gate. The real server blocks here; the fake - must already have a banked credit when the gate asks (auto_free, or a - test's own free_group) — anything else is the deadlock the real - watchdog would kill, so fail loudly. Not appended to ``order``: it is - pacing, not a lifecycle milestone.""" - assert self._pending_freed, ( - "wait_freed with nothing freed: the gather loop would deadlock" - ) - freed = self._pending_freed - self._pending_freed = [] - return freed - - def end_sync(self): - self.order.append("end") - freed = self._pending_freed - self._pending_freed = [] - return freed - - def set_gather_error(self, message): - self.order.append("error") - - -def _rdt_engine_with_fake_server( - source, *, is_sender, client, server, monkeypatch, fleet_owned=None -): - """Build a ShardedRDTTrainerWeightTransferEngine wired to an in-process fake - server (no Ray, no CUDA IPC): bypass trainer_init's spawn, set the - group-major metadata, and route _rpc to the fake.""" - import vllm.distributed.weight_transfer.sharded_rdt_trainer as mod - - # reduce_tensor needs CUDA; the fake server never rebuilds, so stub it. - monkeypatch.setattr(mod, "reduce_tensor", lambda t: (None, ("fake",))) - - init_info = ShardedRDTTrainerInitInfo(num_consumers=1, rank=0 if is_sender else 1) - engine = ShardedRDTTrainerWeightTransferEngine( - client=client, source=source, is_sender=is_sender, init_info=init_info - ) - engine._meta = list(source.metadata()) - names = [m.name for m in engine._meta] - engine._groups = layerwise_groups(names) - engine._server = server - engine._rpc = lambda method, *args: getattr(server, method)(*args) - # What trainer_init resolves from the source's held names + the fleet's - # all-gather. ``fleet_owned`` stands in for that all-gather so a partial- - # ownership rank can be tested without a real process group; the fleet must - # cover every name or the transpose rejects it (nothing would serve the rest). - if fleet_owned is None: - engine._resolve_ownership(1, 0) - else: - # Stands in for the (metadata digest, held-name bitmask) all-gather: - # each rank's group list becomes the mask its held names would set. - names = [m.name for m in engine._meta] - groups = engine._groups - - def _fake_gather(_world, mine): - out = [] - for owned in fleet_owned: - held = {n for gi in owned for n in groups[gi]} - mask = bytearray((len(names) + 7) // 8) - for i, n in enumerate(names): - if n in held: - mask[i >> 3] |= 1 << (i & 7) - out.append((mine[0], bytes(mask))) - return out - - monkeypatch.setattr(engine, "_all_gather_owned", _fake_gather) - engine._resolve_ownership(len(fleet_owned), 0) - return engine - - -def _rdt_source_two_layers(): - return _ListSource( - [ - ("embed.weight", torch.zeros(2)), - ("model.layers.0.w", torch.zeros(2)), - ("model.layers.1.w", torch.zeros(2)), - ("norm.weight", torch.zeros(2)), - ] - ) - - -class TestShardedRDTTrainerInitInfo: - def test_declares_backend(self): - assert ShardedRDTTrainerInitInfo.backend == "sharded_rdt" - - def test_rank_is_keyword_only_and_drives_is_sender(self): - assert ShardedRDTTrainerInitInfo(num_consumers=4, rank=0).is_sender is True - assert ShardedRDTTrainerInitInfo(num_consumers=4, rank=1).is_sender is False - with pytest.raises(TypeError): - # rank is keyword-only. - ShardedRDTTrainerInitInfo(4, 0) # type: ignore[misc] - - def test_registered_in_trainer_factory(self): - cls = WeightTransferTrainerFactory._registry["sharded_rdt"]() - assert cls is ShardedRDTTrainerWeightTransferEngine - - -def test_sharded_rdt_trainer_init_requires_source(): - with pytest.raises(ValueError, match="requires a WeightSource"): - ShardedRDTTrainerWeightTransferEngine.trainer_init( - ShardedRDTTrainerInitInfo(num_consumers=1, rank=0), - client=RecordingClient(), - source=None, - ) - - -def test_sharded_rdt_worker_init_info_is_group_major(monkeypatch): - source = _rdt_source_two_layers() - engine = _rdt_engine_with_fake_server( - source, - is_sender=True, - client=RecordingClient(), - server=_FakeProducerServer(), - monkeypatch=monkeypatch, - ) - worker_init = engine._build_worker_init_info(["srv_rk0"]) - # 4 params -> pre / layer0 / layer1 / post = 4 groups of length 1. - assert worker_init.names == [ - "embed.weight", - "model.layers.0.w", - "model.layers.1.w", - "norm.weight", - ] - assert worker_init.group_lens == [1, 1, 1, 1] - assert worker_init.trainer_actor_names == ["srv_rk0"] - assert worker_init.produce_method_name == "rdt_produce_weights_batched" - assert sum(worker_init.group_lens) == len(worker_init.names) - - -@pytest.mark.skipif( - not torch.cuda.is_available(), - reason="the gather loop's CUDA-IPC export needs a device", -) -def test_sharded_rdt_send_weights_drives_client_in_order(monkeypatch): - server = _FakeProducerServer(auto_free=True) - client = RecordingClient() - engine = _rdt_engine_with_fake_server( - _rdt_source_two_layers(), - is_sender=True, - client=client, - server=server, - monkeypatch=monkeypatch, - ) - engine.send_weights() - - assert client.order == ["start", "update", "finish"] - # begin, one publish per group (4), end. - assert server.order == ["begin", "publish", "publish", "publish", "publish", "end"] - assert len(server.published) == 4 - # every group freed -> no engine-held refs remain. - assert engine._inflight == {} - - -def test_sharded_rdt_send_weights_group_order_mismatch_raises(monkeypatch): - # Source whose iteration order disagrees with its metadata order. - class _BadSource(_ListSource): - def __iter__(self): - reordered = list(self._pairs) - reordered[0], reordered[1] = reordered[1], reordered[0] - return iter(reordered) - - server = _FakeProducerServer() - engine = _rdt_engine_with_fake_server( - _BadSource(_rdt_source_two_layers()._pairs), - is_sender=True, - client=RecordingClient(), - server=server, - monkeypatch=monkeypatch, - ) - with pytest.raises(RuntimeError, match="iteration order must match"): - engine.send_weights() - assert "error" in server.order # gather error propagated to the server - - -@pytest.mark.skipif( - not torch.cuda.is_available(), - reason="the gather loop's CUDA-IPC export needs a device", -) -def test_sharded_rdt_non_sender_skips_client(monkeypatch): - class _RaisingClient(RecordingClient): - def start_weight_update(self): - raise AssertionError("non-sender must not touch the client") - - def update_weights(self, update_info): - raise AssertionError("non-sender must not touch the client") - - def finish_weight_update(self, weight_version: str | None = None): - raise AssertionError("non-sender must not touch the client") - - server = _FakeProducerServer(auto_free=True) - client = _RaisingClient() - engine = _rdt_engine_with_fake_server( - _rdt_source_two_layers(), - is_sender=False, - client=client, - server=server, - monkeypatch=monkeypatch, - ) - engine.send_weights() # gathers only; must not raise - assert client.order == [] - assert server.order == ["begin", "publish", "publish", "publish", "publish", "end"] - - -@pytest.mark.skipif( - not torch.cuda.is_available(), - reason="the gather loop's CUDA-IPC export needs a device", -) -def test_sharded_rdt_send_weights_surfaces_update_error(monkeypatch): - class _FailingUpdateClient(RecordingClient): - def update_weights(self, update_info): - self.order.append("update") - raise RuntimeError("inference side rejected update") - - server = _FakeProducerServer(auto_free=True) - engine = _rdt_engine_with_fake_server( - _rdt_source_two_layers(), - is_sender=True, - client=_FailingUpdateClient(), - server=server, - monkeypatch=monkeypatch, - ) - with pytest.raises(RuntimeError, match="inference side rejected update"): - engine.send_weights() - - -class _OwnedSource(_ListSource): - """A source holding only some groups' names, like a pipeline-parallel rank. - Takes group indices for convenience and declares the names inside them.""" - - def __init__(self, pairs, owned_group_idx): - super().__init__(pairs) - self._owned = list(owned_group_idx) - groups = layerwise_groups([n for n, _ in pairs]) - self._owned_names = [n for gi in self._owned for n in groups[gi]] - - def held_names(self): - return list(self._owned_names) - - def __iter__(self): - by_name = dict(self._pairs) - return iter([(n, by_name[n]) for n in self._owned_names]) - - -@pytest.mark.skipif( - not torch.cuda.is_available(), - reason="the gather loop's CUDA-IPC export needs a device", -) -def test_sharded_rdt_publishes_only_the_groups_it_holds(monkeypatch): - server = _FakeProducerServer(auto_free=True) - engine = _rdt_engine_with_fake_server( - _OwnedSource(_rdt_source_two_layers()._pairs, [1, 2]), - is_sender=False, - client=RecordingClient(), - server=server, - monkeypatch=monkeypatch, - fleet_owned=[[1, 2], [0, 3]], # this rank holds the layers, rank 1 the rest - ) - engine.send_weights() - - assert server.published == [1, 2] - assert server.order == ["begin", "publish", "publish", "end"] - assert engine._inflight == {} - # embed + norm on rank 1, the two layers here: two distinct owner sets, - # numbered by first appearance in metadata order. - assert engine._owner_sets == [[1], [0]] - assert engine._name_owner_class == [0, 1, 1, 0] - - -def test_sharded_rdt_owned_group_order_mismatch_raises(monkeypatch): - class _MisorderedOwned(_OwnedSource): - def __iter__(self): - return iter(list(super().__iter__())[::-1]) - - server = _FakeProducerServer(auto_free=True) - engine = _rdt_engine_with_fake_server( - _MisorderedOwned(_rdt_source_two_layers()._pairs, [1, 2]), - is_sender=False, - client=RecordingClient(), - server=server, - monkeypatch=monkeypatch, - fleet_owned=[[1, 2], [0, 3]], - ) - with pytest.raises(RuntimeError, match="iteration order must match"): - engine.send_weights() - assert "error" in server.order - - -@pytest.mark.skipif( - not torch.cuda.is_available(), - reason="the gather loop's CUDA-IPC export needs a device", -) -def test_sharded_rdt_begin_sync_carries_the_live_count(monkeypatch): - """The free barrier's target is one integer per sync: the live consumer - count, defaulting to the whole provisioned fleet.""" - server = _FakeProducerServer(auto_free=True) - engine = _rdt_engine_with_fake_server( - _rdt_source_two_layers(), - is_sender=False, - client=RecordingClient(), - server=server, - monkeypatch=monkeypatch, - ) - engine.send_weights() - assert server.live_count == 1 # num_consumers=1 in the harness - engine.send_weights(live_consumer_ids=[0]) - assert server.live_count == 1 - - -class TestLiveCountPlumbing: - """``send_weights(live_consumer_ids)`` -> the barrier target. The - provisioned geometry is frozen; a degraded sync only lowers the target and - narrows each slot-sharing group's rendezvous.""" - - @staticmethod - def _engine(num_consumers, world=2, rank=0): - engine = ShardedRDTTrainerWeightTransferEngine.__new__( - ShardedRDTTrainerWeightTransferEngine - ) - engine._init_info = ShardedRDTTrainerInitInfo( - rank=rank, num_consumers=num_consumers - ) - engine.source = object() # send_weights asserts a source is present - received: list = [] - engine._send_weights_inner = lambda count, ids: received.append((count, ids)) - return engine, received - - def test_none_counts_the_whole_provisioned_fleet(self): - engine, got = self._engine(num_consumers=8) - engine.send_weights(None) - assert got == [(8, list(range(8)))] - - def test_a_live_set_counts_its_distinct_members(self): - engine, got = self._engine(num_consumers=8) - engine.send_weights([0, 1, 4, 5, 5]) - assert got == [(4, [0, 1, 4, 5])] - - def test_a_full_live_set_matches_the_provisioned_count(self): - engine, got = self._engine(num_consumers=8) - engine.send_weights(list(range(8))) - engine.send_weights(None) - assert got == [(8, list(range(8)))] * 2 - - def test_which_consumers_died_does_not_matter_only_how_many(self): - """The whole point of the barrier: no routed per-producer targets, so - the identity of the dead consumer is irrelevant to the producers.""" - counts = [] - for live in ([0, 1, 2, 3], [4, 5, 6, 7], [0, 2, 4, 6]): - engine, got = self._engine(num_consumers=8) - engine.send_weights(live) - counts += [c for c, _ids in got] - assert counts == [4, 4, 4] - - def test_the_live_ids_travel_with_the_count(self): - """The count sizes the free barrier, the ids size the slot-sharing - rendezvous, so a producer that shares slots can tell WHICH consumers it - is still waiting for. They must describe the same set.""" - engine, got = self._engine(num_consumers=8) - engine.send_weights([6, 0, 2, 2]) - ((count, ids),) = got - assert ids == [0, 2, 6] and count == len(ids) - - -def test_sharded_rdt_worker_init_info_carries_the_ownership_table(monkeypatch): - import json - - server = _FakeProducerServer(auto_free=True) - engine = _rdt_engine_with_fake_server( - _OwnedSource(_rdt_source_two_layers()._pairs, [0, 1]), - is_sender=True, - client=RecordingClient(), - server=server, - monkeypatch=monkeypatch, - fleet_owned=[[0, 1], [2, 3]], - ) - worker_init = engine._build_worker_init_info(["srv_rk0", "srv_rk1"]) - assert worker_init.owner_sets == [[0], [1]] - assert worker_init.name_owner_class == [0, 0, 1, 1] - assert len(worker_init.name_owner_class) == len(worker_init.names) - # The payload crosses the control plane as JSON: nested lists must survive. - round_tripped = json.loads(json.dumps(asdict(worker_init))) - assert round_tripped["owner_sets"] == [[0], [1]] - assert round_tripped["name_owner_class"] == [0, 0, 1, 1] - - -def test_weight_source_holds_everything_by_default(): - """The contract's default: a source that says nothing holds the whole model.""" - src = _rdt_source_two_layers() - assert src.held_names() is None - assert ModuleSource(torch.nn.Linear(2, 2)).held_names() is None - - -def test_sharded_rdt_rejects_a_held_name_outside_metadata(monkeypatch): - class _BadHeld(_OwnedSource): - def held_names(self): - return ["embed.weight", "not.a.real.name"] - - with pytest.raises(ValueError, match="not"): - _rdt_engine_with_fake_server( - _BadHeld(_rdt_source_two_layers()._pairs, [0]), - is_sender=False, - client=RecordingClient(), - server=_FakeProducerServer(), - monkeypatch=monkeypatch, - ) - - -def test_sharded_rdt_rejects_a_rank_holding_nothing(monkeypatch): - class _HoldsNothing(_ListSource): - def held_names(self): - return [] - - with pytest.raises(ValueError, match="empty"): - _rdt_engine_with_fake_server( - _HoldsNothing(_rdt_source_two_layers()._pairs), - is_sender=False, - client=RecordingClient(), - server=_FakeProducerServer(), - monkeypatch=monkeypatch, - ) - - -def test_sharded_rdt_rejects_a_name_no_rank_holds(monkeypatch): - """Every name must be held somewhere or it can never be served — caught - when the holdings are transposed, naming the orphan.""" - with pytest.raises(ValueError, match="no trainer rank holds"): - _rdt_engine_with_fake_server( - _OwnedSource(_rdt_source_two_layers()._pairs, [0, 1]), - is_sender=False, - client=RecordingClient(), - server=_FakeProducerServer(), - monkeypatch=monkeypatch, - fleet_owned=[[0, 1], [2]], # nobody holds group 3 - ) - - -def test_sharded_rdt_rejects_metadata_disagreement_across_ranks(monkeypatch): - """Only the sender's metadata reaches the consumers, so a rank describing - just its own share must fail loudly rather than silently drop the rest.""" - engine = _rdt_engine_with_fake_server( - _OwnedSource(_rdt_source_two_layers()._pairs, [1, 2]), - is_sender=False, - client=RecordingClient(), - server=_FakeProducerServer(auto_free=True), - monkeypatch=monkeypatch, - fleet_owned=[[1, 2], [0, 3]], # a covering fleet, so construction succeeds - ) - # Now rank 1 reports a DIFFERENT metadata digest for the same model. - monkeypatch.setattr( - engine, - "_all_gather_owned", - lambda w, mine: [mine, ("deadbeefdeadbeef", mine[1])], - ) - with pytest.raises(ValueError, match="disagrees across trainer ranks"): - engine._resolve_ownership(2, 0) - - -def _serve_ring_server(src_name, src): - """A producer server with one cached tensor and a pre-seeded serve ring, so a - pull needs no Ray and no NIXL registration. Returns (server, serve) where - ``serve(chain)`` packs one spec into the SAME ring slot every time — which is - what puts the destination-view cache, and only it, under test.""" - from vllm.distributed.weight_transfer.sharded_rdt_trainer import ( - _RDTProducerServer, - ) - - srv = _RDTProducerServer( - num_rdt_buffers=2, - buffer_presize_gb=0.0, - gather_lookahead=2, - ) - srv._cache[src_name] = src - srv._serve_rings[0] = [ - torch.empty(1 << 16, dtype=torch.uint8, device="cuda") for _ in range(2) - ] - - # The slot is ``seq % nring``, so stepping seq by nring keeps every pull in - # slot 0 while still making each one its own generation -- reusing a seq - # would rendezvous with the finished pack and return it unchanged. - seqs = itertools.count(0, srv._nring) - - def serve(chain): - return srv.rdt_produce_weights_batched( - [(src_name, chain)], consumer_id=0, seq=next(seqs) - )[0] - - return srv, serve - - -@pytest.mark.skipif( - not torch.cuda.is_available(), reason="the producer server needs a CUDA device" -) -def test_serve_does_not_reuse_packed_views_of_another_shape(): - """Two requests can share a name yet pack different slices of it. - - The producer caches the destination views it carves into a serve ring slot. - Keyed by name alone, the second request is packed through the first's views. - Reachable when one name's copies split across owner-class chunks. - """ - name = "model.layers.0.w" - src = torch.arange(64, dtype=torch.bfloat16, device="cuda").reshape(8, 8) - _srv, serve = _serve_ring_server(name, src) - - serve((("narrow", (0, 0, 2), ()),)) # 2 rows - wide = src.narrow(0, 0, 6) # 6 rows, same name, same slot - blob = serve((("narrow", (0, 0, 6), ()),)) - - got = blob[: wide.numel() * wide.element_size()].view(wide.dtype).reshape(6, 8) - assert torch.equal(got, wide) - - -@pytest.mark.skipif( - not torch.cuda.is_available(), reason="the producer server needs a CUDA device" -) -def test_serve_does_not_reuse_packed_views_of_another_dtype(): - """The SILENT case of the same cache: two requests whose slices have the same - name and the same shape but different dtypes pack at identical offsets, so - reusing the stale views raises nothing — ``copy_`` just casts, and the blob - carries the wrong bytes with no check downstream. - - ``view(dtype)`` rather than ``to(dtype)`` because the chain is replayed under - ``ALLOWED_OPS``, which rejects ``to``: same-itemsize dtypes keep the byte - length identical, which is exactly what makes the stale view fit. - """ - name = "model.layers.0.w" - src = torch.arange(64, dtype=torch.bfloat16, device="cuda").reshape(8, 8) - _srv, serve = _serve_ring_server(name, src) - - serve((("view", (torch.float16,), ()),)) # same shape and bytes, fp16 - blob = serve(()) # same name and shape, bf16 - - got = blob[: src.numel() * src.element_size()].view(src.dtype).reshape(8, 8) - assert torch.equal(got, src), "packed through a cached view of the wrong dtype" diff --git a/tests/distributed/test_weight_transfer.py b/tests/distributed/test_weight_transfer.py deleted file mode 100644 index 642193ce88..0000000000 --- a/tests/distributed/test_weight_transfer.py +++ /dev/null @@ -1,2399 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Tests for weight transfer engine backends. - -Unit tests for engine classes (parsing, validation, registry). -Integration tests for NCCL and IPC weight transfer between processes using Ray. -""" - -import builtins -import importlib.util -import pickle -import runpy -import threading -import time -from contextlib import nullcontext -from types import SimpleNamespace -from unittest.mock import MagicMock - -import pybase64 as base64 -import pytest -import ray -import torch -from torch.multiprocessing.reductions import reduce_tensor - -from vllm.config.parallel import ParallelConfig -from vllm.config.weight_transfer import WeightTransferConfig -from vllm.distributed.weight_transfer import ( - HTTPVLLMWeightSyncClient, - ModuleSource, - ParamMeta, - RayVLLMWeightSyncClient, - TrainerWeightTransferEngine, - VLLMWeightSyncClient, - WeightSource, - WeightTransferEngineFactory, - WeightTransferTrainerFactory, -) -from vllm.distributed.weight_transfer.base import ( - TrainerInitInfo, - WeightTransferEngine, - WeightTransferInitRequest, - WeightTransferUpdateRequest, - layerwise_groups, -) -from vllm.distributed.weight_transfer.ipc_engine import ( - IPCTrainerInitInfo, - IPCTrainerWeightTransferEngine, - IPCWeightTransferEngine, - IPCWeightTransferInitInfo, - IPCWeightTransferUpdateInfo, -) -from vllm.distributed.weight_transfer.nccl_engine import ( - NCCLTrainerInitInfo, - NCCLTrainerWeightTransferEngine, - NCCLWeightTransferEngine, - NCCLWeightTransferInitInfo, - NCCLWeightTransferUpdateInfo, -) -from vllm.distributed.weight_transfer.packed_tensor import ( - DEFAULT_PACKED_BUFFER_SIZE_BYTES, - DEFAULT_PACKED_NUM_BUFFERS, -) -from vllm.distributed.weight_transfer.sparse_nccl_engine import ( - SparseNCCLTrainerInitInfo, - SparseNCCLTrainerWeightTransferEngine, - SparseNCCLWeightTransferEngine, - SparseNCCLWeightTransferUpdateInfo, - SparseWeightPatch, -) -from vllm.platforms import current_platform -from vllm.utils.nccl import _nccl_has_no_cache -from vllm.utils.network_utils import get_open_port - - -def _init_ray_for_weight_transfer() -> None: - if ray.is_initialized(): - return - ray.init( - ignore_reinit_error=True, - runtime_env={ - "env_vars": { - "RAY_EXPERIMENTAL_NOSET_CUDA_VISIBLE_DEVICES": "1", - "RAY_EXPERIMENTAL_NOSET_HIP_VISIBLE_DEVICES": "1", - "RAY_EXPERIMENTAL_NOSET_ROCR_VISIBLE_DEVICES": "1", - } - }, - ) - - -def _get_ray_assigned_device() -> torch.device: - gpu_ids = ray.get_gpu_ids() - if not gpu_ids: - return torch.device("cuda:0") - return torch.device(f"cuda:{int(gpu_ids[0])}") - - -def _set_ray_assigned_device() -> torch.device: - device = _get_ray_assigned_device() - current_platform.set_device(device) - return device - - -def create_mock_parallel_config( - rank: int = 0, - world_size: int = 1, - dp_rank: int = 0, -) -> ParallelConfig: - """Create a mock ParallelConfig for testing.""" - config = MagicMock(spec=ParallelConfig) - config.rank = rank - config.world_size = world_size - config.data_parallel_rank = dp_rank - config.data_parallel_index = dp_rank - return config - - -def create_mock_vllm_config( - rank: int = 0, - world_size: int = 1, - dp_rank: int = 0, -) -> MagicMock: - """Create a mock VllmConfig exposing parallel_config and model_config.""" - vllm_config = MagicMock() - vllm_config.parallel_config = create_mock_parallel_config(rank, world_size, dp_rank) - vllm_config.model_config = MagicMock() - return vllm_config - - -# --- Unit Tests: NCCLWeightTransferUpdateInfo Validation --- - - -class TestNCCLWeightTransferUpdateInfoValidation: - """Test NCCLWeightTransferUpdateInfo dataclass validation.""" - - def test_valid_update_info(self): - info = NCCLWeightTransferUpdateInfo( - names=["layer.weight", "layer.bias"], - dtype_names=["float32", "float32"], - shapes=[[10, 10], [10]], - ) - assert info.names == ["layer.weight", "layer.bias"] - assert info.dtype_names == ["float32", "float32"] - assert info.shapes == [[10, 10], [10]] - - def test_mismatched_dtype_names_raises(self): - with pytest.raises(ValueError, match="dtype_names"): - NCCLWeightTransferUpdateInfo( - names=["layer.weight", "layer.bias"], - dtype_names=["float32"], # Only one dtype - shapes=[[10, 10], [10]], - ) - - def test_mismatched_shapes_raises(self): - with pytest.raises(ValueError, match="shapes"): - NCCLWeightTransferUpdateInfo( - names=["layer.weight", "layer.bias"], - dtype_names=["float32", "float32"], - shapes=[[10, 10]], # Only one shape - ) - - def test_empty_lists_valid(self): - info = NCCLWeightTransferUpdateInfo(names=[], dtype_names=[], shapes=[]) - assert len(info.names) == 0 - - -# --- Unit Tests: SparseNCCLWeightTransferUpdateInfo Validation --- - - -class TestSparseNCCLWeightTransferUpdateInfoValidation: - """Test SparseNCCLWeightTransferUpdateInfo dataclass validation.""" - - def test_valid_sparse_update_info(self): - info = SparseNCCLWeightTransferUpdateInfo( - names=["layer.weight", "layer.bias"], - dtype_names=["float32", "bfloat16"], - shapes=[[10, 10], [10]], - num_updates_list=[4, 2], - ) - assert info.num_updates_list == [4, 2] - - def test_mismatched_dtype_names_raises(self): - with pytest.raises(ValueError, match="dtype_names"): - SparseNCCLWeightTransferUpdateInfo( - names=["layer.weight", "layer.bias"], - dtype_names=["float32"], - shapes=[[10, 10], [10]], - num_updates_list=[4, 2], - ) - - def test_rejects_empty_num_updates_list(self): - with pytest.raises(ValueError, match="cannot be empty"): - SparseNCCLWeightTransferUpdateInfo( - names=[], - dtype_names=[], - shapes=[], - num_updates_list=[], - ) - - def test_rejects_mismatched_num_updates(self): - with pytest.raises(ValueError, match="`num_updates_list`"): - SparseNCCLWeightTransferUpdateInfo( - names=["layer.weight", "layer.bias"], - dtype_names=["float32", "float32"], - shapes=[[10, 10], [10]], - num_updates_list=[3], - ) - - def test_rejects_negative_num_updates(self): - with pytest.raises(ValueError, match="non-negative"): - SparseNCCLWeightTransferUpdateInfo( - names=["layer.weight"], - dtype_names=["float32"], - shapes=[[10, 10]], - num_updates_list=[-1], - ) - - -# --- Unit Tests: Engine Parsing --- - - -@pytest.mark.parametrize( - "engine_type", - [ - NCCLWeightTransferEngine, - NCCLTrainerWeightTransferEngine, - SparseNCCLWeightTransferEngine, - SparseNCCLTrainerWeightTransferEngine, - ], -) -def test_nccl_shutdown_destroys_communicator_once(engine_type): - engine = object.__new__(engine_type) - group = MagicMock() - engine.model_update_group = group - - engine.shutdown() - engine.shutdown() - - group.destroy.assert_called_once_with() - assert engine.model_update_group is None - - -class TestNCCLEngineParsing: - """Test NCCLWeightTransferEngine parsing methods.""" - - def _make_engine(self): - config = WeightTransferConfig(backend="nccl") - return NCCLWeightTransferEngine( - config, - create_mock_vllm_config(), - torch.device("cuda"), - MagicMock(spec=torch.nn.Module), - ) - - def test_parse_init_info_valid(self): - engine = self._make_engine() - init_info = engine.parse_init_info( - { - "master_address": "127.0.0.1", - "master_port": 12345, - "rank_offset": 1, - "world_size": 3, - } - ) - assert isinstance(init_info, NCCLWeightTransferInitInfo) - assert init_info.master_address == "127.0.0.1" - assert init_info.master_port == 12345 - assert init_info.rank_offset == 1 - assert init_info.world_size == 3 - - def test_parse_init_info_missing_field_raises(self): - engine = self._make_engine() - with pytest.raises(ValueError, match="Invalid init_info"): - engine.parse_init_info({"master_address": "127.0.0.1"}) - - def test_parse_update_info_valid(self): - engine = self._make_engine() - update_info = engine.parse_update_info( - { - "names": ["w1", "w2"], - "dtype_names": ["float32", "bfloat16"], - "shapes": [[100, 100], [50]], - } - ) - assert isinstance(update_info, NCCLWeightTransferUpdateInfo) - assert update_info.names == ["w1", "w2"] - assert update_info.dtype_names == ["float32", "bfloat16"] - assert update_info.shapes == [[100, 100], [50]] - - -# --- Unit Tests: Engine Registry --- - - -class TestEngineRegistry: - """Test weight transfer engine registry.""" - - def test_create_engine_nccl(self): - config = WeightTransferConfig(backend="nccl") - engine = WeightTransferEngineFactory.create_engine( - config, - create_mock_vllm_config(), - torch.device("cuda"), - MagicMock(spec=torch.nn.Module), - ) - assert isinstance(engine, NCCLWeightTransferEngine) - - def test_create_engine_ipc(self): - config = WeightTransferConfig(backend="ipc") - engine = WeightTransferEngineFactory.create_engine( - config, - create_mock_vllm_config(), - torch.device("cuda"), - MagicMock(spec=torch.nn.Module), - ) - assert isinstance(engine, IPCWeightTransferEngine) - - def test_create_engine_sparse_nccl(self): - config = WeightTransferConfig(backend="sparse_nccl") - engine = WeightTransferEngineFactory.create_engine( - config, - create_mock_vllm_config(), - torch.device("cuda"), - MagicMock(spec=torch.nn.Module), - ) - assert isinstance(engine, SparseNCCLWeightTransferEngine) - - def test_modelexpress_registration_is_native_and_lazy(self, monkeypatch): - from vllm.distributed.weight_transfer import factory - - imported = [] - module = MagicMock() - module.ModelExpressWeightTransferEngine.__name__ = ( - "ModelExpressWeightTransferEngine" - ) - - def import_module(name): - imported.append(name) - return module - - monkeypatch.setattr(factory.importlib, "import_module", import_module) - config = WeightTransferConfig(backend="modelexpress") - vllm_config = create_mock_vllm_config() - device = torch.device("cpu") - model = MagicMock(spec=torch.nn.Module) - - assert imported == [] - engine = WeightTransferEngineFactory.create_engine( - config, vllm_config, device, model - ) - - assert imported == ["vllm.distributed.weight_transfer.modelexpress_engine"] - module.ModelExpressWeightTransferEngine.assert_called_once_with( - config, vllm_config, device, model - ) - assert engine is module.ModelExpressWeightTransferEngine.return_value - - def test_create_engine_invalid_backend(self): - config = WeightTransferConfig(backend="invalid") - with pytest.raises(ValueError, match="Invalid weight transfer backend"): - WeightTransferEngineFactory.create_engine( - config, - create_mock_vllm_config(), - torch.device("cuda"), - MagicMock(spec=torch.nn.Module), - ) - - def test_register_duplicate_raises(self): - with pytest.raises(ValueError, match="already registered"): - WeightTransferEngineFactory.register_engine( - "nccl", NCCLWeightTransferEngine - ) - - -# --- Test receive_weights without init raises --- - - -def test_nccl_receive_weights_without_init_raises(): - """Test that receive_weights raises if init_transfer_engine wasn't called.""" - if torch.accelerator.device_count() < 1: - pytest.skip("Need at least 1 GPU for this test") - - config = WeightTransferConfig(backend="nccl") - engine = NCCLWeightTransferEngine( - config, - create_mock_vllm_config(), - torch.device("cuda"), - MagicMock(spec=torch.nn.Module), - ) - - update_info = NCCLWeightTransferUpdateInfo( - names=["w"], dtype_names=["float32"], shapes=[[10]] - ) - - with pytest.raises(RuntimeError, match="not initialized"): - engine.receive_weights(update_info) - - -def test_sparse_nccl_receive_weights_without_init_raises(): - """Test that sparse receive raises if init_transfer_engine wasn't called.""" - if torch.accelerator.device_count() < 1: - pytest.skip("Need at least 1 GPU for this test") - - config = WeightTransferConfig(backend="sparse_nccl") - engine = SparseNCCLWeightTransferEngine( - config, - create_mock_vllm_config(), - torch.device("cuda"), - MagicMock(spec=torch.nn.Module), - ) - - update_info = SparseNCCLWeightTransferUpdateInfo( - names=["w"], - dtype_names=["float32"], - shapes=[[10]], - num_updates_list=[2], - ) - - with pytest.raises(RuntimeError, match="not initialized"): - engine.receive_weights(update_info) - - -# --- Integration Test: NCCL Weight Transfer Between Ray Tasks --- - - -@ray.remote(num_gpus=1) -def trainer_broadcast_tensor( - master_address: str, - master_port: int, - world_size: int, - tensor_shape: list[int], - tensor_dtype: str, -) -> bool: - """Trainer task that broadcasts a tensor via NCCL.""" - import torch - - device = _set_ray_assigned_device() - - from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator - from vllm.distributed.utils import StatelessProcessGroup - - # Create process group as rank 0 (trainer) - pg = StatelessProcessGroup.create( - host=master_address, - port=master_port, - rank=0, - world_size=world_size, - ) - comm = PyNcclCommunicator(pg, device=device.index) - - # Create and broadcast the tensor - dtype = getattr(torch, tensor_dtype) - tensor_to_send = torch.ones(tensor_shape, dtype=dtype, device=device) - comm.broadcast(tensor_to_send, src=0, stream=torch.cuda.current_stream()) - torch.accelerator.synchronize() - - return True - - -# max_calls=1: a batch-invariant run leaves NCCL pins in the worker process. -@ray.remote(num_gpus=1, max_calls=1) -def inference_receive_tensor( - master_address: str, - master_port: int, - world_size: int, - tensor_shape: list[int], - tensor_dtype: str, - batch_invariant: bool = False, -) -> dict: - """Inference task that receives tensor via NCCLWeightTransferEngine.""" - import contextlib - from unittest.mock import MagicMock - - import torch - - _set_ray_assigned_device() - if batch_invariant: - from vllm.model_executor.determinism.batch_invariant import ( - override_envs_for_invariance, - ) - - override_envs_for_invariance() - # Like vLLM's own groups, a pinned communicator makes NCCL read the - # pins before the transfer group exists. - torch.distributed.init_process_group( - "nccl", - init_method=f"tcp://127.0.0.1:{get_open_port()}", - rank=0, - world_size=1, - ) - torch.distributed.all_reduce(torch.ones(1, device="cuda")) - - from vllm.config.parallel import ParallelConfig - from vllm.config.weight_transfer import WeightTransferConfig - from vllm.distributed.weight_transfer.nccl_engine import ( - NCCLWeightTransferEngine, - NCCLWeightTransferInitInfo, - NCCLWeightTransferUpdateInfo, - ) - - class Recorder(torch.nn.Module): - def __init__(self): - super().__init__() - self.received = [] - - def load_weights(self, weights): - for name, tensor in weights: - self.received.append((name, tensor.clone())) - - config = WeightTransferConfig(backend="nccl") - vllm_config = MagicMock() - parallel_config = MagicMock(spec=ParallelConfig) - parallel_config.rank = 0 - parallel_config.world_size = 1 - parallel_config.data_parallel_rank = 0 - parallel_config.data_parallel_index = 0 - vllm_config.parallel_config = parallel_config - vllm_config.model_config = MagicMock() - - recorder = Recorder() - engine = NCCLWeightTransferEngine( - config, vllm_config, torch.device("cuda"), recorder - ) - # Transport-only test: bypass the set_current_vllm_config context that - # receive_weights enters, since vllm_config here is a mock. - import vllm.config as _vllm_config_mod - - _vllm_config_mod.set_current_vllm_config = lambda cfg: contextlib.nullcontext() - - # Initialize the engine (joins as rank 1) - # Trainer broadcasts a single tensor unpacked, so the worker must not - # expect the packed wire format (packed is a must-agree wire param shipped - # on the init info). - init_info = NCCLWeightTransferInitInfo( - master_address=master_address, - master_port=master_port, - rank_offset=1, # Trainer is rank 0, we become rank 1 - world_size=world_size, - packed=False, - ) - engine.init_transfer_engine(init_info) - - update_info = NCCLWeightTransferUpdateInfo( - names=["test.weight"], - dtype_names=[tensor_dtype], - shapes=[tensor_shape], - ) - engine.receive_weights(update_info) - torch.accelerator.synchronize() - - # Verify we received the tensor - success = False - received_shape = None - received_sum = None - - if len(recorder.received) == 1: - name, tensor = recorder.received[0] - received_shape = list(tensor.shape) - received_sum = tensor.sum().item() - if received_shape == tensor_shape: - expected_sum = 1.0 * torch.tensor(tensor_shape).prod().item() - if abs(received_sum - expected_sum) < 0.01: - success = True - - engine.shutdown() - - return { - "success": success, - "received_shape": received_shape, - "received_sum": received_sum, - } - - -@pytest.mark.skipif( - torch.accelerator.device_count() < 2, - reason="Need at least 2 GPUs to run NCCL weight transfer test.", -) -@pytest.mark.parametrize( - "batch_invariant", - [ - pytest.param(False, id="default"), - pytest.param( - True, - id="batch-invariant-worker", - marks=pytest.mark.skipif( - not _nccl_has_no_cache(), reason="Needs CUDA NCCL >= 2.29.7." - ), - ), - ], -) -def test_nccl_weight_transfer_between_processes(batch_invariant): - """Test NCCL weight transfer from trainer to inference process using Ray. - - This test verifies that the NCCLWeightTransferEngine can receive - tensors broadcast by a trainer process via NCCL, including when the - worker runs with batch-invariance NCCL pins the trainer does not have. - """ - _init_ray_for_weight_transfer() - - master_address = "127.0.0.1" - master_port = get_open_port() - world_size = 2 # 1 trainer + 1 inference worker - - tensor_shape = [100, 100] - tensor_dtype = "float32" - - inference_future = inference_receive_tensor.remote( - master_address, - master_port, - world_size, - tensor_shape, - tensor_dtype, - batch_invariant, - ) - trainer_future = trainer_broadcast_tensor.remote( - master_address, master_port, world_size, tensor_shape, tensor_dtype - ) - - # A mismatched NCCL config deadlocks instead of failing. - trainer_result, result = ray.get([trainer_future, inference_future], timeout=300) - - assert trainer_result, "Trainer should complete successfully" - assert result["success"], ( - f"Weight transfer failed. " - f"Received shape: {result['received_shape']}, " - f"Received sum: {result['received_sum']}" - ) - - -def test_sparse_nccl_checkpoint_chunks_to_ep_local_experts_cpu(monkeypatch): - """Replay global expert patches through two EP-local loaders on CPU.""" - - class CaptureClient: - def __init__(self): - self.order: list[str] = [] - self.update_infos: list[dict] = [] - - def start_weight_update(self): - self.order.append("start") - - def update_weights(self, update_info): - self.order.append("update") - self.update_infos.append(update_info) - - def finish_weight_update(self): - self.order.append("finish") - - monkeypatch.setattr(torch.cuda, "current_stream", lambda *_, **__: None) - monkeypatch.setattr(torch.cuda, "is_available", lambda: False) - monkeypatch.setattr(torch.accelerator, "device_index", lambda *_: nullcontext()) - - client = CaptureClient() - sender = SparseNCCLTrainerWeightTransferEngine(client=client) - sender.model_update_group = MagicMock() - sender.model_update_group.device = torch.device("cpu") - wire_payloads = [] - sender.model_update_group.broadcast.side_effect = lambda tensor, **_: ( - wire_payloads.append(tensor.clone()) - ) - - expert_names = [ - "model.layers.0.mlp.experts.0.gate_proj.weight", - "model.layers.0.mlp.experts.1.gate_proj.weight", - ] - client.start_weight_update() - for name, index, value in zip( - expert_names, - (0, 3), - (5.0, 7.0), - strict=True, - ): - sender.send_weight_chunk( - [ - SparseWeightPatch( - name=name, - indices=torch.tensor([index], dtype=torch.int32), - values=torch.tensor([value]), - full_shape=(2, 2), - ) - ] - ) - client.finish_weight_update() - - assert client.order == ["start", "update", "update", "finish"] - assert [info["names"] for info in client.update_infos] == [ - [expert_names[0]], - [expert_names[1]], - ] - assert [info["shapes"] for info in client.update_infos] == [ - [[2, 2]], - [[2, 2]], - ] - assert [info["num_updates_list"] for info in client.update_infos] == [ - [1], - [1], - ] - - expected_gates = ( - [[5.0, -1.0], [-1.0, -1.0]], - [[-1.0, -1.0], [-1.0, 7.0]], - ) - for ep_rank, expected_gate in enumerate(expected_gates): - model = torch.nn.Module() - model.register_parameter( - "w13_weight", - torch.nn.Parameter(torch.full((1, 4, 2), -1.0), requires_grad=False), - ) - local_name = expert_names[ep_rank] - seen_names: list[str] = [] - loaded_names: set[str] = set() - copy_count = [0] - - def load_weights( - weights, - local_name=local_name, - target=model, - seen=seen_names, - loaded=loaded_names, - count=copy_count, - ): - for name, checkpoint_weight in weights: - seen.append(name) - if name != local_name: - continue - target.w13_weight.data[0, :2].copy_(checkpoint_weight) - loaded.add(name) - count[0] += 1 - return loaded - - model.load_weights = load_weights - receiver = SparseNCCLWeightTransferEngine( - WeightTransferConfig(backend="sparse_nccl"), - create_mock_vllm_config(rank=ep_rank, world_size=2), - torch.device("cpu"), - model, - ) - receiver.model_update_group = MagicMock() - receiver.model_update_group.device = torch.device("cpu") - payloads = iter(wire_payloads) - receiver.model_update_group.broadcast.side_effect = ( - lambda tensor, payloads=payloads, **_: tensor.copy_(next(payloads)) - ) - - receiver.start_weight_update() - for update_info in client.update_infos: - receiver.receive_weights(SparseNCCLWeightTransferUpdateInfo(**update_info)) - receiver.finish_weight_update() - - assert seen_names == expert_names - assert loaded_names == {local_name} - assert copy_count == [1] - assert model.w13_weight[0, :2].tolist() == expected_gate - assert model.w13_weight[0, 2:].eq(-1).all() - foreign_value = 7.0 if ep_rank == 0 else 5.0 - assert not model.w13_weight.eq(foreign_value).any() - receiver.shutdown() - - sender.shutdown() - - -# --- Unit Tests: IPCWeightTransferUpdateInfo Validation --- - - -class TestIPCWeightTransferUpdateInfoValidation: - """Test IPCWeightTransferUpdateInfo dataclass validation.""" - - def test_valid_update_info(self): - if torch.accelerator.device_count() < 1: - pytest.skip("Need at least 1 GPU for this test") - - dummy_tensor = torch.ones(10, 10, device="cuda:0") - _, ipc_handle = reduce_tensor(dummy_tensor) - gpu_uuid = str(torch.cuda.get_device_properties(0).uuid) - ipc_handles = [{gpu_uuid: ipc_handle}] - - info = IPCWeightTransferUpdateInfo( - names=["layer.weight"], - dtype_names=["float32"], - shapes=[[10, 10]], - ipc_handles=ipc_handles, - ) - assert info.names == ["layer.weight"] - assert info.dtype_names == ["float32"] - assert info.shapes == [[10, 10]] - assert len(info.ipc_handles) == 1 - - def test_mismatched_dtype_names_raises(self): - if torch.accelerator.device_count() < 1: - pytest.skip("Need at least 1 GPU for this test") - - dummy_tensor = torch.ones(10, 10, device="cuda:0") - _, ipc_handle = reduce_tensor(dummy_tensor) - gpu_uuid = str(torch.cuda.get_device_properties(0).uuid) - ipc_handles = [{gpu_uuid: ipc_handle}, {gpu_uuid: ipc_handle}] - - with pytest.raises(ValueError, match="dtype_names"): - IPCWeightTransferUpdateInfo( - names=["layer.weight", "layer.bias"], - dtype_names=["float32"], # Only one dtype - shapes=[[10, 10], [10]], - ipc_handles=ipc_handles, - ) - - def test_mismatched_shapes_raises(self): - if torch.accelerator.device_count() < 1: - pytest.skip("Need at least 1 GPU for this test") - - dummy_tensor = torch.ones(10, 10, device="cuda:0") - _, ipc_handle = reduce_tensor(dummy_tensor) - gpu_uuid = str(torch.cuda.get_device_properties(0).uuid) - ipc_handles = [{gpu_uuid: ipc_handle}, {gpu_uuid: ipc_handle}] - - with pytest.raises(ValueError, match="shapes"): - IPCWeightTransferUpdateInfo( - names=["layer.weight", "layer.bias"], - dtype_names=["float32", "float32"], - shapes=[[10, 10]], # Only one shape - ipc_handles=ipc_handles, - ) - - def test_mismatched_ipc_handles_raises(self): - if torch.accelerator.device_count() < 1: - pytest.skip("Need at least 1 GPU for this test") - - dummy_tensor = torch.ones(10, 10, device="cuda:0") - _, ipc_handle = reduce_tensor(dummy_tensor) - gpu_uuid = str(torch.cuda.get_device_properties(0).uuid) - ipc_handles = [{gpu_uuid: ipc_handle}] # Only one handle - - with pytest.raises(ValueError, match="ipc_handles"): - IPCWeightTransferUpdateInfo( - names=["layer.weight", "layer.bias"], - dtype_names=["float32", "float32"], - shapes=[[10, 10], [10]], - ipc_handles=ipc_handles, - ) - - def test_valid_update_info_from_pickled(self, monkeypatch): - if torch.accelerator.device_count() < 1: - pytest.skip("Need at least 1 GPU for this test") - - monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1") - - dummy_tensor = torch.ones(10, 10, device="cuda:0") - ipc_handle = reduce_tensor(dummy_tensor) - gpu_uuid = str(torch.cuda.get_device_properties(0).uuid) - ipc_handles = [{gpu_uuid: ipc_handle}] - - pickled = base64.b64encode(pickle.dumps(ipc_handles)).decode("utf-8") - - info = IPCWeightTransferUpdateInfo( - names=["layer.weight"], - dtype_names=["float32"], - shapes=[[10, 10]], - ipc_handles_pickled=pickled, - ) - assert info.ipc_handles == ipc_handles - assert info.ipc_handles_pickled is None - - def test_pickled_requires_insecure_serialization_flag(self, monkeypatch): - monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "0") - - with pytest.raises(ValueError, match="VLLM_ALLOW_INSECURE_SERIALIZATION=1"): - IPCWeightTransferUpdateInfo( - names=[], - dtype_names=[], - shapes=[], - ipc_handles_pickled=base64.b64encode(pickle.dumps([])).decode("utf-8"), - ) - - def test_both_handles_and_pickled_raises(self): - if torch.accelerator.device_count() < 1: - pytest.skip("Need at least 1 GPU for this test") - - dummy_tensor = torch.ones(10, 10, device="cuda:0") - ipc_handle = reduce_tensor(dummy_tensor) - gpu_uuid = str(torch.cuda.get_device_properties(0).uuid) - ipc_handles = [{gpu_uuid: ipc_handle}] - - pickled = base64.b64encode(pickle.dumps(ipc_handles)).decode("utf-8") - - with pytest.raises(ValueError, match="Cannot specify both"): - IPCWeightTransferUpdateInfo( - names=["layer.weight"], - dtype_names=["float32"], - shapes=[[10, 10]], - ipc_handles=ipc_handles, - ipc_handles_pickled=pickled, - ) - - def test_neither_handles_nor_pickled_raises(self): - with pytest.raises(ValueError, match="must be provided"): - IPCWeightTransferUpdateInfo( - names=["layer.weight"], - dtype_names=["float32"], - shapes=[[10, 10]], - ) - - def test_empty_lists_valid(self): - info = IPCWeightTransferUpdateInfo( - names=[], - dtype_names=[], - shapes=[], - ipc_handles=[], - ) - assert len(info.names) == 0 - - -# --- Unit Tests: IPC Engine Parsing --- - - -class TestIPCEngineParsing: - """Test IPCWeightTransferEngine parsing methods.""" - - def _make_engine(self): - config = WeightTransferConfig(backend="ipc") - return IPCWeightTransferEngine( - config, - create_mock_vllm_config(), - torch.device("cuda"), - MagicMock(spec=torch.nn.Module), - ) - - def test_parse_update_info_valid(self): - if torch.accelerator.device_count() < 1: - pytest.skip("Need at least 1 GPU for this test") - - engine = self._make_engine() - - dummy_tensor1 = torch.ones(100, 100, device="cuda:0") - dummy_tensor2 = torch.ones(50, device="cuda:0") - _, ipc_args1 = reduce_tensor(dummy_tensor1) - _, ipc_args2 = reduce_tensor(dummy_tensor2) - gpu_uuid = str(torch.cuda.get_device_properties(0).uuid) - ipc_handles = [{gpu_uuid: ipc_args1}, {gpu_uuid: ipc_args2}] - - update_info = engine.parse_update_info( - { - "names": ["w1", "w2"], - "dtype_names": ["float32", "bfloat16"], - "shapes": [[100, 100], [50]], - "ipc_handles": ipc_handles, - } - ) - - assert isinstance(update_info, IPCWeightTransferUpdateInfo) - assert update_info.names == ["w1", "w2"] - assert update_info.dtype_names == ["float32", "bfloat16"] - assert update_info.shapes == [[100, 100], [50]] - assert len(update_info.ipc_handles) == 2 - - def test_parse_update_info_pickled(self, monkeypatch): - if torch.accelerator.device_count() < 1: - pytest.skip("Need at least 1 GPU for this test") - - monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1") - - engine = self._make_engine() - - dummy_tensor1 = torch.ones(100, 100, device="cuda:0") - dummy_tensor2 = torch.ones(50, device="cuda:0") - _, ipc_args1 = reduce_tensor(dummy_tensor1) - _, ipc_args2 = reduce_tensor(dummy_tensor2) - gpu_uuid = str(torch.cuda.get_device_properties(0).uuid) - ipc_handles = [{gpu_uuid: ipc_args1}, {gpu_uuid: ipc_args2}] - - pickled = base64.b64encode(pickle.dumps(ipc_handles)).decode("utf-8") - - update_info = engine.parse_update_info( - { - "names": ["w1", "w2"], - "dtype_names": ["float32", "bfloat16"], - "shapes": [[100, 100], [50]], - "ipc_handles_pickled": pickled, - } - ) - - assert isinstance(update_info, IPCWeightTransferUpdateInfo) - assert update_info.names == ["w1", "w2"] - assert len(update_info.ipc_handles) == 2 - assert gpu_uuid in update_info.ipc_handles[0] - assert gpu_uuid in update_info.ipc_handles[1] - - def test_parse_update_info_ignores_none_pickled_handles(self): - engine = self._make_engine() - ipc_handles = [{"gpu-uuid": ("ipc-args",)}] - - update_info = engine.parse_update_info( - { - "names": ["w1"], - "dtype_names": ["float32"], - "shapes": [[1]], - "ipc_handles": ipc_handles, - "ipc_handles_pickled": None, - } - ) - - assert isinstance(update_info, IPCWeightTransferUpdateInfo) - assert update_info.ipc_handles == ipc_handles - - def test_parse_update_info_both_handles_and_pickled_raises(self): - if torch.accelerator.device_count() < 1: - pytest.skip("Need at least 1 GPU for this test") - - engine = self._make_engine() - - dummy_tensor = torch.ones(10, 10, device="cuda:0") - _, ipc_handle = reduce_tensor(dummy_tensor) - gpu_uuid = str(torch.cuda.get_device_properties(0).uuid) - ipc_handles = [{gpu_uuid: ipc_handle}] - - pickled = base64.b64encode(pickle.dumps(ipc_handles)).decode("utf-8") - - with pytest.raises(ValueError, match="Cannot specify both"): - engine.parse_update_info( - { - "names": ["layer.weight"], - "dtype_names": ["float32"], - "shapes": [[10, 10]], - "ipc_handles": ipc_handles, - "ipc_handles_pickled": pickled, - } - ) - - -# --- Integration Test: IPC Weight Transfer Between Ray Tasks --- - - -def get_physical_gpu_id(device_index: int = 0) -> str: - """Get physical GPU UUID for a device.""" - props = torch.cuda.get_device_properties(device_index) - return str(props.uuid) - - -@ray.remote(num_gpus=0.5) -class TrainerActor: - """Trainer actor that creates and holds CUDA IPC handles.""" - - def __init__(self, tensor_shape: list[int], tensor_dtype: str): - device = _set_ray_assigned_device() - - # Create tensor on GPU and keep it alive - dtype = getattr(torch, tensor_dtype) - self.tensor = torch.ones(tensor_shape, dtype=dtype, device=device) - self.tensor.fill_(42.0) # Fill with 42 to verify correct transfer - - _, ipc_args = reduce_tensor(self.tensor) - gpu_uuid = get_physical_gpu_id(device.index) - - torch.accelerator.synchronize() - - self.ipc_handle_dict = { - "ipc_handle": ipc_args, - "gpu_uuid": gpu_uuid, - "shape": tensor_shape, - "dtype": tensor_dtype, - } - - def get_ipc_handle_dict(self) -> dict: - """Return IPC handle dict. Tensor stays alive in this actor.""" - return self.ipc_handle_dict - - -@ray.remote(num_gpus=0.5) -def inference_receive_ipc_tensor( - ipc_handle_dict: dict, - mode: str = "ray", -) -> dict: - """Inference task that receives tensor via IPCWeightTransferEngine.""" - import contextlib - import os - - # Worker-side: ipc_handles_pickled is deserialized via pickle. - if mode == "http": - os.environ["VLLM_ALLOW_INSECURE_SERIALIZATION"] = "1" - - from unittest.mock import MagicMock - - import torch - - device = _set_ray_assigned_device() - - from vllm.config.parallel import ParallelConfig - from vllm.config.weight_transfer import WeightTransferConfig - from vllm.distributed.weight_transfer.ipc_engine import ( - IPCWeightTransferEngine, - ) - - class Recorder(torch.nn.Module): - def __init__(self): - super().__init__() - self.received = [] - - def load_weights(self, weights): - for name, tensor in weights: - self.received.append((name, tensor.clone())) - - # Trainer sends unpacked IPC handles; the worker learns packed=False from - # the init handshake below (IPCWeightTransferInitInfo defaults to False). - config = WeightTransferConfig(backend="ipc") - vllm_config = MagicMock() - parallel_config = MagicMock(spec=ParallelConfig) - parallel_config.rank = 0 - parallel_config.world_size = 1 - parallel_config.data_parallel_rank = 0 - parallel_config.data_parallel_index = 0 - vllm_config.parallel_config = parallel_config - vllm_config.model_config = MagicMock() - - recorder = Recorder() - engine = IPCWeightTransferEngine(config, vllm_config, device, recorder) - # Transport-only test: bypass the set_current_vllm_config context that - # receive_weights enters, since vllm_config here is a mock. - import vllm.config as _vllm_config_mod - - _vllm_config_mod.set_current_vllm_config = lambda cfg: contextlib.nullcontext() - - init_info = IPCWeightTransferInitInfo() - engine.init_transfer_engine(init_info) - - ipc_handles = [{ipc_handle_dict["gpu_uuid"]: ipc_handle_dict["ipc_handle"]}] - - if mode == "ray": - update_dict: dict = { - "names": ["test.weight"], - "dtype_names": [ipc_handle_dict["dtype"]], - "shapes": [ipc_handle_dict["shape"]], - "ipc_handles": ipc_handles, - } - elif mode == "http": - pickled = base64.b64encode(pickle.dumps(ipc_handles)).decode("utf-8") - update_dict = { - "names": ["test.weight"], - "dtype_names": [ipc_handle_dict["dtype"]], - "shapes": [ipc_handle_dict["shape"]], - "ipc_handles_pickled": pickled, - } - else: - raise ValueError(f"Unknown mode: {mode}") - - update_info = engine.parse_update_info(update_dict) - engine.receive_weights(update_info) - torch.accelerator.synchronize() - - success = False - received_shape = None - received_sum = None - - if len(recorder.received) == 1: - name, tensor = recorder.received[0] - received_shape = list(tensor.shape) - received_sum = tensor.sum().item() - if received_shape == ipc_handle_dict["shape"]: - expected_sum = 42.0 * torch.tensor(ipc_handle_dict["shape"]).prod().item() - if abs(received_sum - expected_sum) < 0.01: - success = True - - engine.shutdown() - - return { - "success": success, - "received_shape": received_shape, - "received_sum": received_sum, - } - - -@pytest.mark.skipif( - torch.accelerator.device_count() < 1, - reason="Need at least 1 GPU to run IPC weight transfer test.", -) -@pytest.mark.parametrize("mode", ["ray", "http"]) -def test_ipc_weight_transfer_between_processes(mode: str): - """Test IPC weight transfer from trainer to inference process using Ray.""" - from ray.util.placement_group import placement_group - from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy - - _init_ray_for_weight_transfer() - - pg = placement_group([{"GPU": 1, "CPU": 2}]) - ray.get(pg.ready()) - - scheduling_strategy = PlacementGroupSchedulingStrategy( - placement_group=pg, - placement_group_capture_child_tasks=True, - ) - - tensor_shape = [100, 100] - tensor_dtype = "float32" - - trainer_actor = TrainerActor.options( # type: ignore[attr-defined] - scheduling_strategy=scheduling_strategy - ).remote(tensor_shape, tensor_dtype) - - ipc_handle_dict = ray.get(trainer_actor.get_ipc_handle_dict.remote()) - - inference_result = ray.get( - inference_receive_ipc_tensor.options( - scheduling_strategy=scheduling_strategy - ).remote(ipc_handle_dict, mode=mode) - ) - - assert inference_result["success"], ( - f"IPC weight transfer failed (mode={mode}). " - f"Received shape: {inference_result['received_shape']}, " - f"Received sum: {inference_result['received_sum']}" - ) - - -def test_ipc_receive_weights_missing_gpu_uuid_raises(): - """Test that receive_weights raises if GPU UUID not found in IPC handles.""" - if torch.accelerator.device_count() < 1: - pytest.skip("Need at least 1 GPU for this test") - - config = WeightTransferConfig(backend="ipc") - engine = IPCWeightTransferEngine( - config, - create_mock_vllm_config(), - torch.device("cuda:0"), - MagicMock(spec=torch.nn.Module), - ) - # No init handshake here, so the engine keeps its default packed=False. - - dummy_tensor = torch.ones(10, 10, device="cuda:0") - _, ipc_handle = reduce_tensor(dummy_tensor) - wrong_uuid = "wrong-uuid-12345" - ipc_handles = [{wrong_uuid: ipc_handle}] - - update_info = IPCWeightTransferUpdateInfo( - names=["w"], - dtype_names=["float32"], - shapes=[[10, 10]], - ipc_handles=ipc_handles, - ) - - with pytest.raises(ValueError, match="IPC handle not found"): - engine.receive_weights(update_info) - - -class RecordingClient: - """A fake VLLMWeightSyncClient that records the order of calls.""" - - def __init__(self): - self.order: list[str] = [] - self.last_init_info: dict | None = None - self.last_update_info: dict | None = None - - def init_weight_transfer_engine(self, init_info: dict) -> None: - self.order.append("init") - self.last_init_info = init_info - - def start_weight_update(self) -> None: - self.order.append("start") - - def update_weights(self, update_info: dict) -> None: - self.order.append("update") - self.last_update_info = update_info - - def finish_weight_update(self, weight_version: str | None = None) -> None: - self.order.append("finish") - - -def _module_with(*pairs): - """A tiny nn.Module exposing the given (name, tensor) pairs as parameters, - so trainer tests can build a ModuleSource without a real model.""" - module = torch.nn.Module() - for name, tensor in pairs: - module.register_parameter(name, torch.nn.Parameter(tensor, requires_grad=False)) - return module - - -class _DummyTrainerEngine(TrainerWeightTransferEngine): - """Minimal concrete trainer engine to exercise base-class + factory.""" - - @classmethod - def trainer_init(cls, init_info, *, client, source): - return cls(client=client, source=source) - - def send_weights(self): - pass - - -class TestTrainerClients: - """Structural protocol conformance for the built-in clients.""" - - def test_recording_client_is_protocol(self): - assert isinstance(RecordingClient(), VLLMWeightSyncClient) - - def test_http_client_is_protocol(self): - assert isinstance( - HTTPVLLMWeightSyncClient("http://localhost:8000"), VLLMWeightSyncClient - ) - - def test_ray_client_is_protocol(self): - assert isinstance(RayVLLMWeightSyncClient(MagicMock()), VLLMWeightSyncClient) - - def test_ray_client_sends_typed_requests(self, monkeypatch): - """Ray client must hand the actor typed Request objects, not raw dicts.""" - import ray - - monkeypatch.setattr(ray, "get", lambda refs: None) - handle = MagicMock() - client = RayVLLMWeightSyncClient(handle) - - client.init_weight_transfer_engine({"master_addr": "x"}) - (init_req,), _ = handle.init_weight_transfer_engine.remote.call_args - assert isinstance(init_req, WeightTransferInitRequest) - assert init_req.init_info == {"master_addr": "x"} - - client.update_weights({"names": ["w"]}) - (update_req,), _ = handle.update_weights.remote.call_args - assert isinstance(update_req, WeightTransferUpdateRequest) - assert update_req.update_info == {"names": ["w"]} - - client.finish_weight_update("step-42") - handle.finish_weight_update.remote.assert_called_once_with() - handle.update_weight_version.remote.assert_called_once_with("step-42") - - client.shutdown_weight_transfer_engine() - handle.shutdown_weight_transfer_engine.remote.assert_called_once_with() - - def test_http_client_pickles_ipc_handles_for_json(self, monkeypatch): - """HTTP update_weights must encode raw ipc_handles as a base64 pickle.""" - captured = {} - - def fake_post(self, path, json=None): - captured["path"] = path - captured["json"] = json - - monkeypatch.setattr(HTTPVLLMWeightSyncClient, "_post", fake_post) - client = HTTPVLLMWeightSyncClient("http://localhost:8000") - client.update_weights({"names": ["w"], "ipc_handles": [{"gpu": ("args",)}]}) - sent = captured["json"]["update_info"] - assert "ipc_handles" not in sent - assert "ipc_handles_pickled" in sent - assert pickle.loads(base64.b64decode(sent["ipc_handles_pickled"])) == [ - {"gpu": ("args",)} - ] - - client.update_weights([{"ipc_handles": {"gpu": ("args",)}}, {"names": []}]) - sent = captured["json"]["update_info"] - assert sent[1] == {"names": []} - assert pickle.loads(base64.b64decode(sent[0]["ipc_handles_pickled"])) == { - "gpu": ("args",) - } - - def test_http_client_passes_through_nccl_update_info(self, monkeypatch): - """NCCL update_info has only JSON-native fields and passes unchanged.""" - captured = {} - - def fake_post(self, path, json=None): - captured["path"] = path - captured["json"] = json - - monkeypatch.setattr(HTTPVLLMWeightSyncClient, "_post", fake_post) - client = HTTPVLLMWeightSyncClient("http://localhost:8000") - update_info = {"names": ["w"], "dtype_names": ["float32"], "shapes": [[4]]} - client.update_weights(update_info) - assert captured["json"]["update_info"] == update_info - - client.finish_weight_update("step-42") - assert captured["json"] == {"weight_version": "step-42"} - - client.shutdown_weight_transfer_engine() - assert captured == {"path": "shutdown_weight_transfer_engine", "json": None} - - -class TestModuleSource: - """`ModuleSource` metadata vs. materialized iteration (dense, no GPU).""" - - def test_metadata_reads_shape_and_dtype(self): - source = ModuleSource( - _module_with(("w", torch.zeros(2, 3)), ("b", torch.zeros(3))) - ) - meta = source.metadata() - assert [m.name for m in meta] == ["w", "b"] - assert [m.shape for m in meta] == [(2, 3), (3,)] - assert all(m.dtype == torch.float32 for m in meta) - - def test_iteration_yields_materialized_tensors(self): - w = torch.arange(6, dtype=torch.float32).reshape(2, 3) - source = ModuleSource(_module_with(("w", w))) - pairs = list(source) - assert [name for name, _ in pairs] == ["w"] - assert torch.equal(pairs[0][1], w) - - def test_source_is_reiterable(self): - source = ModuleSource(_module_with(("w", torch.zeros(2)))) - assert [n for n, _ in source] == [n for n, _ in source] == ["w"] - - def test_metadata_agrees_with_iteration(self): - """The two channels must line up element-for-element: engines declare - the round from `metadata()` and then send what iteration yields.""" - source = ModuleSource( - _module_with(("w", torch.zeros(2, 3)), ("b", torch.zeros(3))) - ) - meta = source.metadata() - pairs = list(source) - assert [m.name for m in meta] == [name for name, _ in pairs] - assert [m.dtype for m in meta] == [t.dtype for _, t in pairs] - assert [m.shape for m in meta] == [tuple(t.shape) for _, t in pairs] - - -class TestWeightSourceGroupContract: - """`groups()` / `iter_groups()` on the WeightSource ABC. Groups are what - backends gather and free by, so the default must agree with - `layerwise_groups` over `metadata()`, restricted to what this rank holds.""" - - class _Source(WeightSource): - """Minimal source over an ordered (name, tensor) list, optionally holding - only some names (in which case it iterates only their groups).""" - - def __init__(self, names, held=None, reverse=False): - self._pairs = [(n, torch.full((2,), float(i))) for i, n in enumerate(names)] - self._held = held - self._reverse = reverse - - def metadata(self): - return [ParamMeta(n, t.dtype, tuple(t.shape)) for n, t in self._pairs] - - def held_names(self): - return self._held - - def __iter__(self): - pairs = self._pairs - if self._held is not None: - keep = set(self._held) - pairs = [(n, t) for n, t in pairs if n in keep] - return iter(list(reversed(pairs)) if self._reverse else pairs) - - def _source(self, names, held=None, reverse=False): - return self._Source(names, held, reverse) - - def test_groups_defaults_to_the_layerwise_partition(self): - names = ["embed.w", "model.layers.0.a", "model.layers.1.a", "norm.w"] - assert self._source(names).groups() == layerwise_groups(names) - - def test_groups_keeps_only_groups_holding_a_held_name(self): - names = ["embed.w", "model.layers.0.a", "model.layers.1.a", "norm.w"] - held = ["model.layers.0.a", "model.layers.1.a"] - assert self._source(names, held=held).groups() == [ - ["model.layers.0.a"], - ["model.layers.1.a"], - ] - - def test_groups_keeps_a_partially_held_group_whole(self): - """One held name selects the WHOLE group, unheld members included. - - A group is the collective unit, so its membership has to be - rank-uniform: a rank that narrowed it to what it holds would disagree - with its peers about which names group *g* covers, and the gather would - desynchronize. Selection is per group, but only whole groups; the - per-name split is the backend's, via owner sets. This is the - foreign-expert shape under expert parallelism — a rank holds some - experts of a layer, not all of them. - """ - names = ["model.layers.0.a", "model.layers.0.b", "model.layers.1.a"] - held = ["model.layers.0.a"] - assert self._source(names, held=held).groups() == [ - ["model.layers.0.a", "model.layers.0.b"], - ] - - def test_groups_order_follows_metadata_not_the_declaration(self): - """The declaration is a SET of names; the groups it selects still come - out in metadata order, so each pairs with the right ``iter_groups()`` - batch however the source listed them.""" - names = ["embed.w", "model.layers.0.a", "model.layers.1.a", "norm.w"] - source = self._source( - names, held=["model.layers.1.a", "model.layers.0.a", "model.layers.1.a"] - ) - assert source.groups() == [["model.layers.0.a"], ["model.layers.1.a"]] - assert [ns for ns, _ in source.iter_groups()] == [ - ["model.layers.0.a"], - ["model.layers.1.a"], - ] - - def test_iter_groups_batches_the_stream_per_group(self): - names = ["embed.w", "model.layers.0.a", "model.layers.0.b", "norm.w"] - batches = list(self._source(names).iter_groups()) - assert [ns for ns, _ in batches] == [ - ["embed.w"], - ["model.layers.0.a", "model.layers.0.b"], - ["norm.w"], - ] - assert all(len(ns) == len(ts) for ns, ts in batches) - - def test_iter_groups_yields_the_tensors_iteration_produced(self): - names = ["model.layers.0.a", "model.layers.0.b"] - (batch,) = list(self._source(names).iter_groups()) - _names, tensors = batch - assert [float(t[0]) for t in tensors] == [0.0, 1.0] - - def test_iter_groups_yields_only_held_groups(self): - names = ["embed.w", "model.layers.0.a", "model.layers.1.a"] - batches = list(self._source(names, held=["model.layers.1.a"]).iter_groups()) - assert [ns for ns, _ in batches] == [["model.layers.1.a"]] - - def test_out_of_order_iteration_raises(self): - """Materializing is usually a collective, so a rank that iterates out of - order deadlocks its peers -- fail loudly instead.""" - source = self._source(["model.layers.0.a", "model.layers.0.b"], reverse=True) - with pytest.raises(RuntimeError, match="iteration order must match"): - list(source.iter_groups()) - - def test_a_source_may_override_iter_groups(self): - """The extension point: materialize a whole group in one step instead of - one generator resume per tensor.""" - calls = [] - - class _Batched(TestWeightSourceGroupContract._Source): - def iter_groups(self): - for group in self.groups(): - calls.append(len(group)) - yield group, [torch.zeros(2) for _ in group] - - source = _Batched(["model.layers.0.a", "model.layers.0.b"]) - assert [ns for ns, _ in source.iter_groups()] == [ - ["model.layers.0.a", "model.layers.0.b"] - ] - assert calls == [2] - - -class TestDeferredProcessingContract: - """`defers_processing` and `drain_pending` are two halves of one contract: a - caller that takes over the update tail (running its own - `finalize_layerwise_reload` instead of going through `finish_weight_update`) - reads the flag and calls the method. Both must be answerable on any engine, or - that caller ends up reaching through a getattr.""" - - def _engines(self): - registry = dict(WeightTransferEngineFactory._registry) - # ModelExpress is an optional, separately installed package, but its - # backend is always registered. Skip it when the package is missing. - if importlib.util.find_spec("modelexpress") is None: - registry.pop("modelexpress") - return {name: loader() for name, loader in registry.items()} - - def test_every_engine_declares_whether_it_defers(self): - for name, cls in self._engines().items(): - assert isinstance(cls.defers_processing, bool), name - - def test_every_engine_can_be_drained(self): - """The default is a no-op, so a caller never has to check whether the - method exists before calling it.""" - for name, cls in self._engines().items(): - assert callable(cls.drain_pending), name - - def test_the_default_is_not_to_defer(self): - assert WeightTransferEngine.defers_processing is False - - def test_a_synchronous_engine_drains_as_a_no_op(self): - engine = object.__new__(WeightTransferEngineFactory._registry["nccl"]()) - engine.drain_pending() # must not raise, and must not need any state - - def test_the_rdt_engine_defers_and_overrides_the_drain(self): - """The one engine the contract exists for.""" - cls = WeightTransferEngineFactory._registry["sharded_rdt"]() - assert cls.defers_processing is True - assert cls.drain_pending is not WeightTransferEngine.drain_pending - - -class TestTrainerFactory: - """WeightTransferTrainerFactory registry mechanics.""" - - def test_registry_has_all_backends(self): - assert "nccl" in WeightTransferTrainerFactory._registry - assert "ipc" in WeightTransferTrainerFactory._registry - assert "sparse_nccl" in WeightTransferTrainerFactory._registry - - def test_register_and_dispatch(self): - saved = dict(WeightTransferTrainerFactory._registry) - try: - WeightTransferTrainerFactory.register_engine("dummy", _DummyTrainerEngine) - engine = WeightTransferTrainerFactory.trainer_init( - MagicMock(backend="dummy"), # backend read from the init info - client=RecordingClient(), - source=ModuleSource(_module_with(("w", torch.zeros(2)))), - ) - assert isinstance(engine, _DummyTrainerEngine) - with pytest.raises(ValueError, match="already registered"): - WeightTransferTrainerFactory.register_engine( - "dummy", _DummyTrainerEngine - ) - finally: - WeightTransferTrainerFactory._registry = saved - - def test_unknown_backend_raises(self): - with pytest.raises(ValueError, match="Invalid weight transfer backend"): - WeightTransferTrainerFactory.trainer_init( - MagicMock(backend="nope"), - client=RecordingClient(), - source=ModuleSource(_module_with(("w", torch.zeros(2)))), - ) - - def test_ipc_init_info_declares_backend(self): - assert IPCTrainerInitInfo.backend == "ipc" - - def test_nccl_init_info_declares_backend(self): - assert NCCLTrainerInitInfo.backend == "nccl" - - def test_sparse_nccl_init_info_declares_backend(self): - assert SparseNCCLTrainerInitInfo.backend == "sparse_nccl" - - def test_trainer_init_info_subclass_must_set_backend(self): - with pytest.raises(TypeError, match="class-level `backend`"): - - class _NoBackend(TrainerInitInfo): - pass - - -class TestTrainerEngineBase: - """Base-class construction (no GPU).""" - - def test_source_stored_and_sender_by_default(self): - engine = _DummyTrainerEngine( - client=RecordingClient(), - source=ModuleSource(_module_with(("w", torch.zeros(2)))), - ) - assert engine.is_sender is True - assert [name for name, _ in engine.source] == ["w"] - - def test_shutdown_default_is_noop(self): - engine = _DummyTrainerEngine( - client=RecordingClient(), - source=ModuleSource(_module_with(("w", torch.zeros(2)))), - is_sender=False, - ) - assert engine.is_sender is False - engine.shutdown() # must not raise - - -@pytest.mark.skipif( - torch.accelerator.device_count() < 1, - reason="Need at least 1 GPU (CUDA IPC handles).", -) -def test_ipc_trainer_send_weights_drives_client_in_order(): - """send_weights issues start -> update -> finish and ships per-round metadata; - the packed wire param rides the init info, not the per-round update_info.""" - client = RecordingClient() - engine = IPCTrainerWeightTransferEngine( - client=client, - source=ModuleSource(_module_with(("w", torch.ones(4, device="cuda")))), - packed=False, - ) - - engine.send_weights() - - assert client.order == ["start", "update", "finish"] - assert client.last_update_info is not None - assert client.last_update_info["names"] == ["w"] - assert client.last_update_info["shapes"] == [[4]] - assert "packed" not in client.last_update_info - - -def test_ipc_trainer_init_ships_packed_to_worker(): - """trainer_init drives the inference-side init handshake and propagates the - must-agree `packed` flag to the worker.""" - if torch.accelerator.device_count() < 1: - pytest.skip("Need at least 1 GPU (CUDA IPC handles).") - - client = RecordingClient() - engine = WeightTransferTrainerFactory.trainer_init( - init_info=IPCTrainerInitInfo(rank=0, packed=True), # backend from init info - client=client, - source=ModuleSource(_module_with(("w", torch.ones(4, device="cuda")))), - ) - - assert isinstance(engine, IPCTrainerWeightTransferEngine) - assert engine.is_sender is True - assert engine.packed is True - assert client.order == ["init"] - assert client.last_init_info == {"packed": True} - - -def test_nccl_trainer_init_ships_worker_init_info(monkeypatch): - """The sender's trainer_init drives the inference-side init handshake with - the worker-shaped init info (rank_offset=1) while opening its own endpoint, - and propagates the must-agree wire params to the worker.""" - import vllm.distributed.weight_transfer.nccl_engine as nccl_engine_mod - - # Bypass the real NCCL rendezvous. - monkeypatch.setattr( - nccl_engine_mod, "open_trainer_endpoint", lambda info: MagicMock() - ) - - client = RecordingClient() - engine = WeightTransferTrainerFactory.trainer_init( - init_info=NCCLTrainerInitInfo( - master_address="127.0.0.1", - master_port=29500, - world_size=3, - rank=0, - packed=True, - packed_buffer_size_bytes=1024, - packed_num_buffers=3, - ), - client=client, - source=ModuleSource(_module_with(("w", torch.zeros(4)))), - ) - - assert isinstance(engine, NCCLTrainerWeightTransferEngine) - assert engine.is_sender is True - assert engine.packed is True - assert client.order == ["init"] - assert client.last_init_info == { - "master_address": "127.0.0.1", - "master_port": 29500, - "rank_offset": 1, - "world_size": 3, - "packed": True, - "packed_buffer_size_bytes": 1024, - "packed_num_buffers": 3, - } - - -def test_nccl_worker_learns_wire_params_from_init_handshake(monkeypatch): - """The worker engine reads packed + buffer geometry from the - trainer-supplied init info at the handshake, not from the config or the - per-round update info.""" - import vllm.distributed.weight_transfer.nccl_engine as nccl_engine_mod - - monkeypatch.setattr( - nccl_engine_mod, "worker_init_process_group", lambda info, pc: MagicMock() - ) - - engine = NCCLWeightTransferEngine( - WeightTransferConfig(backend="nccl"), - create_mock_vllm_config(), - torch.device("cuda:0"), - MagicMock(spec=torch.nn.Module), - ) - assert engine.packed is False # pre-handshake default (legacy unpacked) - engine.init_transfer_engine( - NCCLWeightTransferInitInfo( - master_address="127.0.0.1", - master_port=29500, - rank_offset=1, - world_size=2, - packed=True, - packed_buffer_size_bytes=2048, - packed_num_buffers=4, - ) - ) - - assert engine.packed is True - assert engine.packed_buffer_size_bytes == 2048 - assert engine.packed_num_buffers == 4 - - -def test_nccl_trainer_init_non_sender_skips_rendezvous_and_client(): - """Non-sender trainer ranks build an engine without opening an endpoint or - touching the client; they only join the collectives in send_weights.""" - client = RecordingClient() - engine = WeightTransferTrainerFactory.trainer_init( - init_info=NCCLTrainerInitInfo( - master_address="127.0.0.1", - master_port=29500, - world_size=3, - rank=1, - ), - client=client, - source=ModuleSource(_module_with(("w", torch.zeros(4)))), - ) - - assert engine.is_sender is False - assert engine.model_update_group is None - assert client.order == [] - - # send_weights on a non-sender only iterates the source (packed mode needs - # no CUDA stream on non-senders), never the client. - engine.send_weights() - assert client.order == [] - - -@pytest.mark.skipif( - torch.accelerator.device_count() < 1, - reason="Need at least 1 GPU (NCCL broadcast / CUDA stream).", -) -def test_nccl_trainer_send_weights_drives_client_in_order(): - """send_weights issues start -> update -> finish and ships per-round - metadata; the packed wire params ride the init handshake, not the - per-round update_info.""" - client = RecordingClient() - engine = NCCLTrainerWeightTransferEngine( - client=client, - source=ModuleSource(_module_with(("w", torch.zeros(4, device="cuda")))), - packed=False, - ) - # Bypass the real NCCL rendezvous; broadcast is a no-op. - engine.model_update_group = MagicMock() - - engine.send_weights() - - assert client.order == ["start", "update", "finish"] - assert client.last_update_info is not None - assert client.last_update_info["names"] == ["w"] - assert client.last_update_info["shapes"] == [[4]] - assert "packed" not in client.last_update_info - - -class _ScriptedSource(WeightSource): - """Declares `meta` but yields whatever `pairs` says — used to drive the - metadata/iteration agreement checks.""" - - def __init__(self, meta, pairs): - self._meta = meta - self._pairs = pairs - - def metadata(self): - return list(self._meta) - - def __iter__(self): - yield from self._pairs - - -def _mock_group_engine(source, monkeypatch, **kwargs): - """Unpacked trainer engine with a mocked group and stream (no GPU needed).""" - engine = NCCLTrainerWeightTransferEngine( - client=RecordingClient(), source=source, packed=False, **kwargs - ) - engine.model_update_group = MagicMock() - monkeypatch.setattr(torch.cuda, "current_stream", MagicMock()) - return engine - - -def test_nccl_trainer_init_requires_source(): - """NCCL is a full-resync backend: it cannot run without a WeightSource.""" - with pytest.raises(ValueError, match="requires a WeightSource"): - NCCLTrainerWeightTransferEngine.trainer_init( - NCCLTrainerInitInfo( - master_address="127.0.0.1", master_port=29500, world_size=2, rank=0 - ), - client=RecordingClient(), - ) - - -def test_nccl_trainer_send_weights_rejects_reordered_source(monkeypatch): - """The worker sizes its buffers (and cuts packed chunks) from metadata(), so - iteration disagreeing with it must raise rather than corrupt the stream.""" - meta = [ - ParamMeta("w", torch.float32, (4,)), - ParamMeta("b", torch.float32, (2,)), - ] - reordered = [("b", torch.zeros(2)), ("w", torch.zeros(4))] - engine = _mock_group_engine(_ScriptedSource(meta, reordered), monkeypatch) - - with pytest.raises(ValueError, match="disagrees with iteration at index 0"): - engine.send_weights() - - -def test_nccl_trainer_send_weights_rejects_dtype_disagreement(monkeypatch): - """A source that declares one wire dtype and materializes another would make - the two sides disagree on every byte offset.""" - meta = [ParamMeta("w", torch.float32, (4,))] - engine = _mock_group_engine( - _ScriptedSource(meta, [("w", torch.zeros(4, dtype=torch.bfloat16))]), - monkeypatch, - ) - - with pytest.raises(ValueError, match="disagrees with iteration"): - engine.send_weights() - - -def test_nccl_trainer_send_weights_rejects_truncated_source(monkeypatch): - """Yielding fewer parameters than declared leaves the worker waiting.""" - meta = [ - ParamMeta("w", torch.float32, (4,)), - ParamMeta("b", torch.float32, (2,)), - ] - engine = _mock_group_engine( - _ScriptedSource(meta, [("w", torch.zeros(4))]), monkeypatch - ) - - with pytest.raises(ValueError, match="yielded 1 parameters"): - engine.send_weights() - - -def test_nccl_trainer_send_weights_broadcasts_contiguous(monkeypatch): - """NCCL sends numel elements from data_ptr(), so a non-contiguous view must - be linearized first or the worker receives unrelated memory.""" - base = torch.arange(6, dtype=torch.float32).reshape(2, 3) - view = base.t() # non-contiguous - meta = [ParamMeta("w", torch.float32, tuple(view.shape))] - engine = _mock_group_engine(_ScriptedSource(meta, [("w", view)]), monkeypatch) - - engine.send_weights() - - sent = engine.model_update_group.broadcast.call_args.args[0] - assert sent.is_contiguous() - assert torch.equal(sent, view) - - -def test_nccl_trainer_send_weights_raises_instead_of_hanging(monkeypatch): - """A failed broadcast must surface even while the inference-side - update_weights is still blocked in its matching NCCL call. - - Joining the RPC thread there would deadlock: the worker only returns once - the broadcast it is waiting for arrives, which never happens. - """ - rpc_entered = threading.Event() - release_rpc = threading.Event() - - class _BlockingClient(RecordingClient): - def update_weights(self, update_info): - rpc_entered.set() - release_rpc.wait(timeout=30) # stands in for a wedged NCCL recv - super().update_weights(update_info) - - class _FailingSource(WeightSource): - def metadata(self): - return [ParamMeta("w", torch.float32, (4,))] - - def __iter__(self): - # Fail only once the RPC is provably in flight. - rpc_entered.wait(timeout=60) - raise RuntimeError("broadcast blew up") - - engine = NCCLTrainerWeightTransferEngine( - client=_BlockingClient(), source=_FailingSource(), packed=False - ) - engine.model_update_group = MagicMock() - monkeypatch.setattr(torch.cuda, "current_stream", MagicMock()) - - started = time.perf_counter() - try: - with pytest.raises(RuntimeError, match="broadcast blew up"): - engine.send_weights() - elapsed = time.perf_counter() - started - assert rpc_entered.is_set(), "the RPC was never in flight" - assert not release_rpc.is_set(), "send_weights waited for the wedged RPC" - # Regression guard: joining the RPC thread would park here until the - # client's own timeout expires instead of raising immediately. - assert elapsed < 5.0, f"send_weights blocked {elapsed:.1f}s on the RPC" - finally: - release_rpc.set() - - -def _sparse_patch(device: str = "cpu") -> SparseWeightPatch: - return SparseWeightPatch( - name="w", - indices=torch.tensor([1, 3], dtype=torch.int32, device=device), - values=torch.tensor([1.0, 2.0], dtype=torch.float32, device=device), - full_shape=(4, 4), - ) - - -def test_sparse_nccl_trainer_init_ships_worker_init_info(monkeypatch): - """The sender's trainer_init drives the init handshake with the - worker-shaped init info; sparse ships no packed wire params, so the worker - keeps its unpacked defaults. Sparse takes no `source`.""" - import vllm.distributed.weight_transfer.sparse_nccl_engine as sparse_mod - - monkeypatch.setattr(sparse_mod, "open_trainer_endpoint", lambda info: MagicMock()) - - client = RecordingClient() - engine = WeightTransferTrainerFactory.trainer_init( - init_info=SparseNCCLTrainerInitInfo( - master_address="127.0.0.1", - master_port=29500, - world_size=2, - rank=0, - ), - client=client, - ) - - assert isinstance(engine, SparseNCCLTrainerWeightTransferEngine) - assert client.order == ["init"] - assert client.last_init_info == { - "master_address": "127.0.0.1", - "master_port": 29500, - "rank_offset": 1, - "world_size": 2, - "packed": False, - "packed_buffer_size_bytes": DEFAULT_PACKED_BUFFER_SIZE_BYTES, - "packed_num_buffers": DEFAULT_PACKED_NUM_BUFFERS, - } - - -def test_sparse_nccl_trainer_send_weights_drives_client_in_order(monkeypatch): - """One-shot send drives start, update, and finish in order.""" - client = RecordingClient() - engine = SparseNCCLTrainerWeightTransferEngine(client=client) - engine.model_update_group = MagicMock() - engine.model_update_group.device = torch.device("cpu") - # The group is a mock, so the stream is just a handle it is handed (and a - # handle _post_send_sync can synchronize). - monkeypatch.setattr(torch.cuda, "current_stream", MagicMock()) - - engine.send_weights([_sparse_patch()]) - - assert client.order == ["start", "update", "finish"] - assert client.last_update_info is not None - assert client.last_update_info["names"] == ["w"] - assert client.last_update_info["shapes"] == [[4, 4]] - assert client.last_update_info["num_updates_list"] == [2] - # One broadcast for indices + one for values per patch. - assert engine.model_update_group.broadcast.call_count == 2 - engine.shutdown() - - -def test_sparse_nccl_trainer_send_weights_empty_round_is_noop(): - """A round with no patches must not touch the client (an empty sparse - update info is invalid by construction).""" - client = RecordingClient() - engine = SparseNCCLTrainerWeightTransferEngine(client=client) - engine.model_update_group = MagicMock() - - engine.send_weights([]) - engine.send_weights() # no argument is also a no-op round - engine.send_weight_chunk([]) - engine.send_weight_chunk() - - assert client.order == [] - - -def test_sparse_nccl_trainer_send_weights_requires_full_shape(): - patch = _sparse_patch() - patch.full_shape = None - engine = SparseNCCLTrainerWeightTransferEngine(client=RecordingClient()) - engine.model_update_group = MagicMock() - engine.model_update_group.device = torch.device("cpu") - - with pytest.raises(ValueError, match="full_shape"): - engine.send_weights([patch]) - - -def test_sparse_nccl_trainer_rejects_source(): - """Sparse is a delta backend; a WeightSource would silently never be sent.""" - with pytest.raises(ValueError, match="takes no WeightSource"): - SparseNCCLTrainerWeightTransferEngine( - client=RecordingClient(), - source=ModuleSource(_module_with(("w", torch.zeros(2)))), - ) - - -def test_sparse_nccl_trainer_validates_patch_before_any_rpc(): - """Malformed patches must fail on the trainer, before start_weight_update: - the worker's own checks only run once the broadcasts are already in flight, - where a size mismatch wedges both sides instead of raising.""" - client = RecordingClient() - engine = SparseNCCLTrainerWeightTransferEngine(client=client) - engine.model_update_group = MagicMock() - engine.model_update_group.device = torch.device("cpu") - - mismatched = SparseWeightPatch( - name="w", - indices=torch.tensor([1, 3], dtype=torch.int32), - values=torch.tensor([1.0], dtype=torch.float32), - full_shape=(4, 4), - ) - with pytest.raises(ValueError, match="matching lengths"): - engine.send_weights([mismatched]) - - wrong_index_dtype = SparseWeightPatch( - name="w", - indices=torch.tensor([1, 3], dtype=torch.int64), - values=torch.tensor([1.0, 2.0], dtype=torch.float32), - full_shape=(4, 4), - ) - with pytest.raises(ValueError, match="int32 indices"): - engine.send_weights([wrong_index_dtype]) - - assert client.order == [] - - -def test_sparse_nccl_trainer_non_sender_skips_client(): - client = RecordingClient() - engine = WeightTransferTrainerFactory.trainer_init( - init_info=SparseNCCLTrainerInitInfo( - master_address="127.0.0.1", - master_port=29500, - world_size=2, - rank=1, - ), - client=client, - ) - - assert engine.is_sender is False - assert engine.model_update_group is None - assert isinstance(engine, SparseNCCLTrainerWeightTransferEngine) - engine.send_weights([_sparse_patch()]) - engine.send_weight_chunk([_sparse_patch()]) - assert client.order == [] - - -# --- Optional ModelExpress Client Lifecycle --- - - -def _import_modelexpress_shim(): - spec = importlib.util.find_spec( - "vllm.distributed.weight_transfer.modelexpress_engine" - ) - assert spec is not None and spec.origin is not None - runpy.run_path(spec.origin, run_name="_test_modelexpress_shim") - - -@pytest.mark.parametrize( - "missing_module", - [ - "modelexpress", - "modelexpress_rl", - "modelexpress_rl.inference", - "modelexpress_rl.inference.engines", - "modelexpress_rl.inference.engines.vllm", - "modelexpress_rl.inference.engines.vllm.weight_transfer_engine", - ], -) -def test_modelexpress_missing_backend_explains_source_install( - monkeypatch, missing_module -): - original_import = builtins.__import__ - error = ModuleNotFoundError(name=missing_module) - - def import_without_backend(name, *args, **kwargs): - if name == "modelexpress": - raise error - return original_import(name, *args, **kwargs) - - monkeypatch.setattr(builtins, "__import__", import_without_backend) - with pytest.raises(ImportError, match="modelexpress_client/python") as exc_info: - _import_modelexpress_shim() - assert exc_info.value.__cause__ is error - assert "uv pip install" in str(exc_info.value) - - -@pytest.mark.parametrize("missing_module", ["grpc", "modelexpress.internal_dependency"]) -def test_modelexpress_preserves_transitive_import_errors(monkeypatch, missing_module): - original_import = builtins.__import__ - error = ModuleNotFoundError(name=missing_module) - - def import_without_dependency(name, *args, **kwargs): - if name == "modelexpress": - raise error - return original_import(name, *args, **kwargs) - - monkeypatch.setattr(builtins, "__import__", import_without_dependency) - with pytest.raises(ModuleNotFoundError) as exc_info: - _import_modelexpress_shim() - assert exc_info.value is error - - -@pytest.fixture -def mx_client(monkeypatch): - client_module = pytest.importorskip("modelexpress_rl.inference.client") - client = MagicMock() - monkeypatch.setattr( - client_module.ModelExpressGeneratorClient, - "initialize", - MagicMock(return_value=client), - ) - monkeypatch.setattr(torch.accelerator, "synchronize", MagicMock()) - return client - - -def _make_modelexpress_engine(): - return WeightTransferEngineFactory.create_engine( - WeightTransferConfig(backend="modelexpress"), - SimpleNamespace( - parallel_config=SimpleNamespace(), - model_config=SimpleNamespace(model="test/model"), - ), - torch.device("cpu"), - torch.nn.Linear(2, 2), - ) - - -@pytest.fixture -def mx_worker(mx_client, monkeypatch): - from vllm.v1.worker import gpu_worker - - monkeypatch.setattr(gpu_worker, "set_current_vllm_config", lambda _: nullcontext()) - worker = object.__new__(gpu_worker.Worker) - worker.weight_transfer_engine = _make_modelexpress_engine() - worker.vllm_config = worker.weight_transfer_engine.vllm_config - worker._weight_update_active = False - worker._weight_update_is_draft = False - worker.model_runner = MagicMock() - return worker - - -def test_modelexpress_worker_rejects_updates_before_initialization( - mx_worker, mx_client -): - with pytest.raises(RuntimeError, match="not initialized"): - mx_worker.start_weight_update() - with pytest.raises(RuntimeError, match="start_weight_update must be called"): - mx_worker.update_weights({"version_id": "version-a"}) - with pytest.raises(RuntimeError, match="without a matching"): - mx_worker.finish_weight_update() - mx_client.stage_weight.assert_not_called() - - -@pytest.mark.parametrize( - "second_update, error_type", - [ - ({"version_id": "version-a"}, RuntimeError), - ({"version_id": "version-b"}, RuntimeError), - ({"version_id": ""}, ValueError), - ({"version_id": None}, ValueError), - ({}, ValueError), - ], -) -def test_modelexpress_worker_recovers_after_rejected_update( - mx_worker, mx_client, second_update, error_type -): - """A rejected payload must release A so a fresh session can actually install B.""" - mx_worker.weight_transfer_engine.init_transfer_engine( - mx_worker.weight_transfer_engine.parse_init_info({}) - ) - staged_a = SimpleNamespace(version_id="version-a", metrics={}, release=MagicMock()) - staged_b = SimpleNamespace(version_id="version-b", metrics={}, release=MagicMock()) - mx_client.stage_weight.side_effect = [staged_a, staged_b] - mx_client.apply_weight.return_value = {} - mx_worker.start_weight_update() - mx_worker.update_weights({"version_id": "version-a"}) - - with pytest.raises(error_type): - mx_worker.update_weights(second_update) - staged_a.release.assert_called_once_with() - mx_client.apply_weight.assert_called_once_with(staged_a) - - mx_worker.start_weight_update() - mx_worker.update_weights({"version_id": "version-b"}) - mx_worker.finish_weight_update() - - assert [ - call.kwargs["version"].version_id - for call in mx_client.stage_weight.call_args_list - ] == ["version-a", "version-b"] - mx_client.apply_weight.assert_called_with(staged_b) - staged_b.release.assert_called_once_with() - - -def test_modelexpress_worker_can_retry_failed_finish(mx_worker, mx_client): - mx_worker.weight_transfer_engine.init_transfer_engine( - mx_worker.weight_transfer_engine.parse_init_info({}) - ) - staged = mx_client.stage_weight.return_value - staged.release.side_effect = [RuntimeError("release failed"), None] - mx_worker.start_weight_update() - mx_worker.update_weights({"version_id": "version-a"}) - - with pytest.raises(RuntimeError, match="release failed"): - mx_worker.finish_weight_update() - mx_worker.finish_weight_update() - - assert staged.release.call_count == 2 - assert not mx_worker._weight_update_active - mx_worker.start_weight_update() - - -def test_native_engine_applies_and_releases_exact_versions(mx_client): - from modelexpress_rl.inference.engines.vllm import ( - weight_transfer_engine as mx_engine, - ) - - from vllm.distributed.weight_transfer import modelexpress_engine - - engine = _make_modelexpress_engine() - assert type(engine) is mx_engine.ModelExpressWeightTransferEngine - assert ( - modelexpress_engine.ModelExpressWeightTransferInitInfo - is mx_engine.ModelExpressWeightTransferInitInfo - ) - assert ( - modelexpress_engine.ModelExpressWeightTransferUpdateInfo - is mx_engine.ModelExpressWeightTransferUpdateInfo - ) - assert not engine.supports_draft_weight_update - engine.init_transfer_engine(engine.parse_init_info({})) - - for version_id in ("version-a", "version-b"): - staged = SimpleNamespace(version_id=version_id, metrics={}, release=MagicMock()) - mx_client.stage_weight.return_value = staged - mx_client.apply_weight.return_value = {} - engine.start_weight_update() - engine.update_weights({"version_id": version_id}) - - assert ( - mx_client.stage_weight.call_args.kwargs["version"].version_id == version_id - ) - mx_client.apply_weight.assert_called_with(staged) - torch.accelerator.synchronize.assert_called() - staged.release.assert_not_called() - engine.finish_weight_update() - staged.release.assert_called_once_with() - - engine.shutdown() - engine.shutdown() - mx_client.close.assert_called_once_with() - - -def test_vime_init_config_reaches_mx_client(mx_client): - from modelexpress_rl.inference.client import ModelExpressGeneratorClient - - engine = _make_modelexpress_engine() - ModelExpressGeneratorClient.initialize.assert_not_called() - engine.init_transfer_engine( - engine.parse_init_info( - { - "model_name": "policy", - "server_url": "mx:8001", - "initial_serving_version_id": "serving-a", - "object_storage_type": "S3", - "initial_base_version_id": "base-a", - "seed_checkpoint_path": "/models/launch", - "refit_checkpoint_dir": "/mxdelta/receiver", - "refit_checkpoint_max_size_gb": 200, - "object_storage_endpoint_url": "http://minio:9000", - "object_storage_region_name": "us-west-2", - "registration_ttl_seconds": 90, - "lease_ttl_seconds": 60, - "max_transfer_attempts": 4, - "max_replay_chain_length": 17, - "rpc_timeout_seconds": 12.5, - } - ) - ) - config = ModelExpressGeneratorClient.initialize.call_args.args[0] - assert config.engine_context.model is engine.model - assert config.engine_context.vllm_config is engine.vllm_config - assert config.model_name == "policy" - assert config.server_url == "mx:8001" - assert config.initial_serving_version_id == "serving-a" - assert config.registration_ttl_seconds == 90 - assert config.lease_ttl_seconds == 60 - assert config.max_transfer_attempts == 4 - assert config.max_replay_chain_length == 17 - assert config.rpc_timeout_seconds == 12.5 - storage = config.object_storage - assert storage.storage_type.value == "S3" - assert storage.initial_base_version_id == "base-a" - assert storage.seed_checkpoint_path == "/models/launch" - assert storage.refit_checkpoint_dir == "/mxdelta/receiver" - assert storage.refit_checkpoint_max_size_gb == 200 - assert storage.endpoint_url == "http://minio:9000" - assert storage.region_name == "us-west-2" - - -@pytest.mark.parametrize( - "key", - [ - "object_storage_type", - "initial_base_version_id", - "seed_checkpoint_path", - "refit_checkpoint_dir", - "object_storage_endpoint_url", - "object_storage_region_name", - ], -) -def test_incomplete_storage_config_rejected_before_init(mx_client, key): - from modelexpress_rl.inference.client import ModelExpressGeneratorClient - - engine = _make_modelexpress_engine() - with pytest.raises(ValueError, match="object storage requires"): - engine.init_transfer_engine(engine.parse_init_info({key: "value"})) - ModelExpressGeneratorClient.initialize.assert_not_called() - - -@pytest.mark.parametrize("version_id", ["", " ", None, 1]) -def test_invalid_version_rejected_before_staging(mx_client, version_id): - engine = _make_modelexpress_engine() - engine.init_transfer_engine(engine.parse_init_info({})) - engine.start_weight_update() - with pytest.raises(ValueError, match="version_id is required"): - engine.update_weights({"version_id": version_id}) - mx_client.stage_weight.assert_not_called() - - -@pytest.mark.parametrize("failure", ["stage", "apply", "release"]) -def test_update_failure_preserves_error_and_releases_handle(mx_client, failure): - engine = _make_modelexpress_engine() - engine.init_transfer_engine(engine.parse_init_info({})) - staged = SimpleNamespace(version_id="version-a", metrics={}, release=MagicMock()) - mx_client.stage_weight.return_value = staged - if failure == "stage": - mx_client.stage_weight.side_effect = RuntimeError("stage failed") - else: - mx_client.apply_weight.side_effect = RuntimeError("apply failed") - if failure == "release": - staged.release.side_effect = RuntimeError("release failed") - engine.start_weight_update() - with pytest.raises(RuntimeError, match="stage failed|apply failed"): - engine.update_weights({"version_id": "version-a"}) - assert staged.release.call_count == (0 if failure == "stage" else 1) - torch.accelerator.synchronize.assert_not_called() - engine.shutdown() - mx_client.close.assert_called_once_with() - - -def test_plugin_does_not_replace_native_backend(mx_client): - from modelexpress.engines.vllm.registration import ( - register_plugin_weight_transfer_engine, - ) - from modelexpress_rl.inference.engines.vllm.weight_transfer_engine import ( - ModelExpressWeightTransferEngine, - ) - - native_loader = WeightTransferEngineFactory._registry["modelexpress"] - register_plugin_weight_transfer_engine() - assert WeightTransferEngineFactory._registry["modelexpress"] is native_loader - assert type(_make_modelexpress_engine()) is ModelExpressWeightTransferEngine diff --git a/tests/distributed/test_weight_transfer_nccl_uid.py b/tests/distributed/test_weight_transfer_nccl_uid.py deleted file mode 100644 index 18a6442a2a..0000000000 --- a/tests/distributed/test_weight_transfer_nccl_uid.py +++ /dev/null @@ -1,465 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Tests for the pre-shared ncclUniqueId weight-transfer rendezvous. - -The rendezvous lets a peer that cannot join a TCPStore (e.g. a torch-free JAX -trainer) initialize NCCL weight transfer from a unique id minted out of band. - -* Unit tests for the mutually-exclusive rendezvous-mode validation on the NCCL - init infos. -* Two-GPU integration tests driving the real worker engine against a peer: one - via vLLM's own ``PyNcclCommunicator``, one via a genuinely torch-free trainer - (cupy buffer + raw ``ncclBroadcast``). - -The shared Ray helpers come from the sibling module (unchanged); the worker task -is local so the sibling's TCP test and its worker stay untouched. -""" - -from unittest.mock import MagicMock - -import pybase64 as base64 -import pytest -import ray -import torch - -from tests.distributed.test_weight_transfer import ( - _init_ray_for_weight_transfer, - _set_ray_assigned_device, - create_mock_vllm_config, -) -from vllm.config.weight_transfer import WeightTransferConfig -from vllm.distributed.weight_transfer.nccl_engine import ( - NCCLTrainerInitInfo, - NCCLWeightTransferEngine, - NCCLWeightTransferInitInfo, -) - -# Standard base64 of exactly 128 zero bytes: a well-formed (dummy) ncclUniqueId. -VALID_UID_B64 = base64.b64encode(b"\x00" * 128).decode() - - -class TestNCCLRendezvousValidation: - """Test the xor-of-rendezvous-modes validation on the NCCL init infos.""" - - def test_worker_init_info_is_keyword_only(self): - # kw_only so a stale positional call fails loudly rather than silently - # swapping fields once the rendezvous fields became optional. - with pytest.raises(TypeError): - NCCLWeightTransferInitInfo(1, 3, "127.0.0.1", 12345) # type: ignore[call-arg] - - def test_trainer_init_info_accepts_positional_fields(self): - # NCCLTrainerInitInfo keeps positional subclass fields; only `rank` - # (from TrainerInitInfo) is keyword-only. - trainer_positional = NCCLTrainerInitInfo("127.0.0.1", 12345, 3, rank=0) - trainer_keyword = NCCLTrainerInitInfo( - master_address="127.0.0.1", - master_port=12345, - world_size=3, - rank=0, - ) - assert trainer_positional == trainer_keyword - - def test_tcp_mode_valid(self): - info = NCCLWeightTransferInitInfo( - rank_offset=1, - world_size=3, - master_address="127.0.0.1", - master_port=12345, - ) - assert info.master_address == "127.0.0.1" - assert info.nccl_unique_id_b64 is None - - def test_uid_mode_valid(self): - info = NCCLWeightTransferInitInfo( - rank_offset=1, - world_size=3, - nccl_unique_id_b64=VALID_UID_B64, - ) - assert info.nccl_unique_id_b64 == VALID_UID_B64 - assert info.master_address is None - assert info.master_port is None - assert VALID_UID_B64 not in repr(info) - - def test_both_modes_raises(self): - with pytest.raises(ValueError, match="not both"): - NCCLWeightTransferInitInfo( - rank_offset=1, - world_size=3, - master_address="127.0.0.1", - master_port=12345, - nccl_unique_id_b64=VALID_UID_B64, - ) - - def test_neither_mode_raises(self): - with pytest.raises(ValueError, match="need nccl_unique_id_b64"): - NCCLWeightTransferInitInfo(rank_offset=1, world_size=3) - - def test_half_specified_tcp_raises(self): - with pytest.raises(ValueError, match="master_port"): - NCCLWeightTransferInitInfo( - rank_offset=1, - world_size=3, - master_address="127.0.0.1", - ) - - def test_bad_base64_uid_raises(self): - with pytest.raises(ValueError, match="not valid standard base64"): - NCCLWeightTransferInitInfo( - rank_offset=1, - world_size=3, - nccl_unique_id_b64="!!!!", - ) - - def test_wrong_length_uid_raises(self): - short_uid = base64.b64encode(b"\x00" * 64).decode() - with pytest.raises(ValueError, match="128"): - NCCLWeightTransferInitInfo( - rank_offset=1, - world_size=3, - nccl_unique_id_b64=short_uid, - ) - - def test_parse_init_info_uid_valid(self): - engine = NCCLWeightTransferEngine( - WeightTransferConfig(backend="nccl"), - create_mock_vllm_config(), - torch.device("cuda"), - MagicMock(spec=torch.nn.Module), - ) - init_info = engine.parse_init_info( - { - "nccl_unique_id_b64": VALID_UID_B64, - "rank_offset": 1, - "world_size": 3, - } - ) - assert isinstance(init_info, NCCLWeightTransferInitInfo) - assert init_info.nccl_unique_id_b64 == VALID_UID_B64 - assert init_info.master_address is None - - def test_parse_init_info_both_modes_raises(self): - engine = NCCLWeightTransferEngine( - WeightTransferConfig(backend="nccl"), - create_mock_vllm_config(), - torch.device("cuda"), - MagicMock(spec=torch.nn.Module), - ) - with pytest.raises(ValueError, match="not both"): - engine.parse_init_info( - { - "master_address": "127.0.0.1", - "master_port": 12345, - "nccl_unique_id_b64": VALID_UID_B64, - "rank_offset": 1, - "world_size": 3, - } - ) - - @pytest.mark.parametrize( - ("unique_id_bytes", "rank", "world_size", "match"), - [ - (b"\x00" * 64, 0, 2, "128-byte"), - (b"\x00" * 128, -1, 2, "out of range"), - (b"\x00" * 128, 2, 2, "out of range"), - ], - ) - def test_from_unique_id_bytes_validation( - self, unique_id_bytes, rank, world_size, match - ): - from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator - - with pytest.raises(ValueError, match=match): - PyNcclCommunicator.from_unique_id_bytes( - unique_id_bytes, - rank=rank, - world_size=world_size, - device=0, - ) - - -@ray.remote(num_gpus=1) -def inference_receive_tensor( - init_info_dict: dict, - tensor_shape: list[int], - tensor_dtype: str, -) -> dict: - """Worker that joins via the given init dict and records the tensor it gets. - - Local to this module (rather than the sibling's TCP-only worker) so it can - take a UID init payload -- the same dict boundary an external trainer POSTs. - """ - import contextlib - - import torch - - _set_ray_assigned_device() - - from vllm.config.parallel import ParallelConfig - from vllm.config.weight_transfer import WeightTransferConfig - from vllm.distributed.weight_transfer.nccl_engine import ( - NCCLWeightTransferEngine, - NCCLWeightTransferInitInfo, - NCCLWeightTransferUpdateInfo, - ) - - class Recorder(torch.nn.Module): - def __init__(self): - super().__init__() - self.received = [] - - def load_weights(self, weights): - for name, tensor in weights: - self.received.append((name, tensor.clone())) - - config = WeightTransferConfig(backend="nccl") - vllm_config = MagicMock() - parallel_config = MagicMock(spec=ParallelConfig) - parallel_config.rank = 0 - parallel_config.world_size = 1 - parallel_config.data_parallel_rank = 0 - parallel_config.data_parallel_index = 0 - vllm_config.parallel_config = parallel_config - vllm_config.model_config = MagicMock() - - recorder = Recorder() - engine = NCCLWeightTransferEngine( - config, vllm_config, torch.device("cuda"), recorder - ) - # Transport-only test: bypass the set_current_vllm_config context that - # receive_weights enters, since vllm_config here is a mock. - import vllm.config as _vllm_config_mod - - _vllm_config_mod.set_current_vllm_config = lambda cfg: contextlib.nullcontext() - - try: - engine.init_transfer_engine(NCCLWeightTransferInitInfo(**init_info_dict)) - engine.receive_weights( - NCCLWeightTransferUpdateInfo( - names=["test.weight"], - dtype_names=[tensor_dtype], - shapes=[tensor_shape], - ) - ) - torch.accelerator.synchronize() - - received_shape = None - received_sum = None - if len(recorder.received) == 1: - _name, tensor = recorder.received[0] - received_shape = list(tensor.shape) - received_sum = tensor.sum().item() - - return { - "success": received_shape == tensor_shape - and abs(received_sum - torch.tensor(tensor_shape).prod().item()) < 0.01, - "received_shape": received_shape, - "received_sum": received_sum, - } - finally: - engine.shutdown() - - -@ray.remote(num_gpus=1) -def trainer_broadcast_tensor_uid( - uid_b64: str, - world_size: int, - tensor_shape: list[int], - tensor_dtype: str, -) -> dict: - """Drive rank 0 via vLLM's own communicator from a pre-shared unique id. - - A real torch-free trainer would mint the id and drive rank 0 itself; here we - reuse the real send protocol by injecting the communicator into a sender - engine, exercising the worker-side UID path end to end. - """ - import pybase64 as base64 - import torch - - _set_ray_assigned_device() - - from vllm.distributed.weight_transfer import ModuleSource - from vllm.distributed.weight_transfer.nccl_common import uid_init_process_group - from vllm.distributed.weight_transfer.nccl_engine import ( - NCCLTrainerWeightTransferEngine, - ) - - class NoopClient: - def init_weight_transfer_engine(self, init_info): - pass - - def start_weight_update(self): - pass - - def update_weights(self, update_info): - pass - - def finish_weight_update(self, weight_version=None): - pass - - dtype = getattr(torch, tensor_dtype) - model = torch.nn.Module() - test_module = torch.nn.Module() - test_module.register_parameter( - "weight", - torch.nn.Parameter( - torch.ones(tensor_shape, dtype=dtype, device="cuda"), requires_grad=False - ), - ) - model.add_module("test", test_module) - - engine = NCCLTrainerWeightTransferEngine( - client=NoopClient(), - source=ModuleSource(model), - is_sender=True, - packed=False, - ) - engine.model_update_group = uid_init_process_group( - base64.b64decode(uid_b64), - rank=0, - world_size=world_size, - device=torch.accelerator.current_device_index(), - ) - try: - engine.send_weights() - torch.accelerator.synchronize() - return {"success": True} - finally: - engine.shutdown() - - -@ray.remote(num_gpus=1) -def trainer_broadcast_tensor_torch_free( - uid_b64: str, - world_size: int, - tensor_shape: list[int], - tensor_dtype: str, -) -> dict: - """Trainer peer that uses no torch and no PyNcclCommunicator (a JAX stand-in). - - Drives NCCL through the raw ctypes wrapper with a cupy send buffer, so it - exercises interop with a foreign NCCL peer rather than vLLM against itself. - ``PyNcclCommunicator.__init__`` runs a one-element warm-up all_reduce right - after ``ncclCommInitRank``; that is a collective, so this peer must issue the - matching all_reduce before the broadcast or both ranks deadlock. - """ - import cupy as cp - import numpy as np - import pybase64 as base64 - - device = _set_ray_assigned_device() - cp.cuda.Device(device.index).use() - - from vllm.distributed.device_communicators.pynccl_wrapper import ( - NCCLLibrary, - buffer_type, - cudaStream_t, - ncclDataTypeEnum, - ncclRedOpTypeEnum, - ) - - assert tensor_dtype == "float32", "harness only wires float32" - # Fill on the host and copy up (a plain H2D memcpy) so we neither use torch - # nor make cupy JIT-compile a fill kernel (which needs NVRTC). All-ones lets - # the worker verify by sum == numel. - sendbuff = cp.asarray(np.ones(tensor_shape, dtype=np.float32)) - warmup = cp.asarray(np.zeros(1, dtype=np.float32)) - numel = int(sendbuff.size) - # The copies above run on the default stream; drain them before the NCCL - # collectives read the buffers on our own stream. - cp.cuda.Device(device.index).synchronize() - stream = cp.cuda.Stream() - - nccl = NCCLLibrary() - comm = nccl.ncclCommInitRank( - world_size, nccl.unique_id_from_bytes(base64.b64decode(uid_b64)), 0 - ) - try: - # Mirror the worker's init-time warm-up all_reduce (rank 0 must join it). - nccl.ncclAllReduce( - buffer_type(warmup.data.ptr), - buffer_type(warmup.data.ptr), - 1, - ncclDataTypeEnum.ncclFloat32, - ncclRedOpTypeEnum.ncclSum, - comm, - cudaStream_t(stream.ptr), - ) - nccl.ncclBroadcast( - buffer_type(sendbuff.data.ptr), - buffer_type(sendbuff.data.ptr), - numel, - ncclDataTypeEnum.ncclFloat32, - 0, # root - comm, - cudaStream_t(stream.ptr), - ) - stream.synchronize() - return {"success": True} - finally: - # Abort (not the collective destroy) for an uncoordinated teardown. - nccl.ncclCommAbort(comm) - - -def _run_uid_transfer(trainer_task) -> None: - """Mint a unique id, run the worker and the given trainer peer concurrently, - and assert the tensor round-trips. Both ranks must enter init together -- - there is no store barrier on the UID path.""" - from vllm.distributed.device_communicators.pynccl_wrapper import NCCLLibrary - - _init_ray_for_weight_transfer() - - nccl = NCCLLibrary() - uid_b64 = base64.b64encode(bytes(nccl.ncclGetUniqueId().internal)).decode() - - world_size = 2 # 1 trainer + 1 inference worker - tensor_shape = [100, 100] - tensor_dtype = "float32" - - inference_future = inference_receive_tensor.remote( - { - "rank_offset": 1, - "world_size": world_size, - "nccl_unique_id_b64": uid_b64, - "packed": False, - }, - tensor_shape, - tensor_dtype, - ) - trainer_future = trainer_task.remote( - uid_b64, world_size, tensor_shape, tensor_dtype - ) - - try: - trainer_result, result = ray.get( - [trainer_future, inference_future], timeout=120 - ) - finally: - ray.shutdown() - - assert trainer_result["success"], "Trainer should complete successfully" - assert result["success"], ( - f"Weight transfer failed. " - f"Received shape: {result['received_shape']}, " - f"Received sum: {result['received_sum']}" - ) - - -@pytest.mark.skipif( - torch.accelerator.device_count() < 2, - reason="Need at least 2 GPUs to run NCCL weight transfer test.", -) -def test_nccl_weight_transfer_between_processes_uid(): - """Weight transfer over a pre-shared ncclUniqueId (no TCPStore), with both - ranks on vLLM's ``PyNcclCommunicator``.""" - _run_uid_transfer(trainer_broadcast_tensor_uid) - - -@pytest.mark.skipif( - torch.accelerator.device_count() < 2, - reason="Need at least 2 GPUs to run NCCL weight transfer test.", -) -def test_nccl_weight_transfer_torch_free_trainer(): - """UID weight transfer against a foreign peer that uses no torch and no vLLM - communicator, only a cupy buffer and raw ``ncclBroadcast``.""" - pytest.importorskip("cupy") - _run_uid_transfer(trainer_broadcast_tensor_torch_free) diff --git a/tests/entrypoints/rl/weight_transfer_metrics/test_across_api_servers_metrics.py b/tests/entrypoints/rl/weight_transfer_metrics/test_across_api_servers_metrics.py deleted file mode 100644 index 108dd77b2f..0000000000 --- a/tests/entrypoints/rl/weight_transfer_metrics/test_across_api_servers_metrics.py +++ /dev/null @@ -1,70 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Weight-transfer metrics scraped from a real two-API-server deployment.""" - -import requests -import torch -from prometheus_client.parser import text_string_to_metric_families -from transformers import AutoModelForCausalLM - -from tests.entrypoints.rl.conftest import MODEL_NAME -from tests.utils import RemoteOpenAIServer -from vllm.distributed.weight_transfer import ( - HTTPVLLMWeightSyncClient, - ModuleSource, - WeightTransferTrainerFactory, -) -from vllm.distributed.weight_transfer.ipc_engine import IPCTrainerInitInfo - -DURATION = "vllm:rl_weight_update_operation_duration_seconds" -IN_FLIGHT = "vllm:rl_weight_update_operations_in_flight" -SYNCS = 8 - - -def scrape(url: str) -> dict[tuple[str, str], float]: - text = requests.get(f"{url}/metrics", timeout=30).text - return { - (sample.name, sample.labels["operation"]): sample.value - for family in text_string_to_metric_families(text) - for sample in family.samples - if sample.name in (f"{DURATION}_count", IN_FLIGHT) - } - - -def test_weight_sync_metrics_aggregate_across_api_servers(tmp_path): - args = [ - "--load-format", - "dummy", - "--enforce-eager", - "--max-model-len", - "512", - "--gpu-memory-utilization", - "0.5", - "--api-server-count", - "2", - "--weight-transfer-config", - '{"backend": "ipc"}', - ] - env = { - "VLLM_SERVER_DEV_MODE": "1", - "VLLM_ALLOW_INSECURE_SERIALIZATION": "1", - "PROMETHEUS_MULTIPROC_DIR": str(tmp_path), - } - with RemoteOpenAIServer(MODEL_NAME, args, env_dict=env) as server: - url = server.url_root - trainer = AutoModelForCausalLM.from_pretrained( - MODEL_NAME, torch_dtype=torch.bfloat16 - ).cuda() - engine = WeightTransferTrainerFactory.trainer_init( - IPCTrainerInitInfo(rank=0), - client=HTTPVLLMWeightSyncClient(url), - source=ModuleSource(trainer), - ) - for _ in range(SYNCS): - engine.send_weights() - metrics = scrape(url) - - calls = {"init": 1, "start": SYNCS, "update": SYNCS, "finish": SYNCS} - for operation, count in calls.items(): - assert metrics[(f"{DURATION}_count", operation)] == count, operation - assert metrics[(IN_FLIGHT, operation)] == 0, operation diff --git a/tests/kernels/attention/test_pack_unpack_triton.py b/tests/kernels/attention/test_pack_unpack_triton.py deleted file mode 100644 index 158ae550ef..0000000000 --- a/tests/kernels/attention/test_pack_unpack_triton.py +++ /dev/null @@ -1,234 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -import torch -from torch.testing import assert_close - -from vllm.v1.attention.ops.common import pack_seq_triton, unpack_seq_triton - - -def test_pack_seq_basic_fp8(): - """Test basic functionality of pack_seq_triton with fp8 and 3D tensors.""" - device = "cuda" - dtype = torch.float8_e4m3fn - - # Test cases with 3D tensors (N, H, D) - test_cases = [ - (6, 8, 4, 2, [3, 3]), # (6, 8, 4) -> (2, 3, 8, 4) - (10, 4, 8, 3, [2, 4, 4]), # (10, 4, 8) -> (3, 4, 4, 8) - (20, 16, 32, 4, [5, 5, 5, 5]), # (20, 16, 32) -> (4, 5, 16, 32) - ] - - for N, H, D, B, lengths_list in test_cases: - # Create input tensor with small values for fp8 - x = torch.randn(N, H, D, dtype=torch.float32, device=device) * 0.1 - x = x.to(dtype=dtype) - lengths = torch.tensor(lengths_list, device=device) - - # Pack the data - packed = pack_seq_triton(x, lengths) - - # Check output shape and properties - expected_shape = (B, max(lengths_list), H, D) - assert packed.shape == expected_shape - assert packed.dtype == dtype - assert packed.device == x.device - - # Check that valid data is preserved (within fp8 precision) - for b in range(B): - start_idx = sum(lengths_list[:b]) - seq_len = lengths_list[b] - - expected_data = x[start_idx : start_idx + seq_len].to(torch.float32) - actual_data = packed[b, :seq_len].to(torch.float32) - - assert_close(actual_data, expected_data, rtol=1e-1, atol=1e-2) - - -def test_pack_seq_custom_padding_fp8(): - """Test pack_seq_triton with custom padding values for fp8.""" - device = "cuda" - dtype = torch.float8_e4m3fn - N, H, D, B = 20, 8, 16, 2 - lengths = torch.tensor([10, 10], device=device) - - x = torch.randn(N, H, D, dtype=torch.float32, device=device) * 0.1 - x = x.to(dtype=dtype) - - # Test with different padding values - for pad_value in [-100.0, -10.0, 0.0, 10.0, 100.0]: - result = pack_seq_triton(x, lengths, pad_value=pad_value) - - # Check valid data - for b in range(B): - start_idx = b * 10 - expected_data = x[start_idx : start_idx + 10].to(torch.float32) - actual_data = result[b, :10].to(torch.float32) - assert_close(actual_data, expected_data, rtol=1e-1, atol=1e-2) - - # Check padding (fp8 has limited range, so check for large values) - padded_data = result[:, 10:].to(torch.float32) - if pad_value < 0: - assert torch.all(padded_data < -50) # Large negative values - elif pad_value > 0: - assert torch.all(padded_data > 50) # Large positive values - else: - assert torch.allclose(padded_data, torch.zeros_like(padded_data), atol=1e-2) - - -def test_pack_seq_default_negative_inf_padding_fp8(): - """Test that pack_seq_triton uses -inf padding by default for fp8.""" - device = "cuda" - dtype = torch.float8_e4m3fn - # B = 2 - N, H, D = 20, 8, 16 - lengths = torch.tensor([10, 10], device=device) - - x = torch.randn(N, H, D, dtype=torch.float32, device=device) * 0.1 - x = x.to(dtype=dtype) - result = pack_seq_triton(x, lengths) - - # Check that padding is large negative values (fp8 representation of -inf) - padded_data = result[:, 10:].to(torch.float32) - assert torch.all( - padded_data < -100 - ) # fp8 -inf is represented as large negative number - - -def test_pack_seq_edge_cases_fp8(): - """Test pack_seq_triton with edge cases for fp8.""" - device = "cuda" - dtype = torch.float8_e4m3fn - - # Test with single batch element - x = torch.randn(10, 8, 16, dtype=torch.float32, device=device) * 0.1 - x = x.to(dtype=dtype) - lengths = torch.tensor([10], device=device) - result = pack_seq_triton(x, lengths) - assert result.shape == (1, 10, 8, 16) - - # Test with very short sequences - x = torch.randn(20, 4, 8, dtype=torch.float32, device=device) * 0.1 - x = x.to(dtype=dtype) - lengths = torch.tensor([1, 1, 1], device=device) - result = pack_seq_triton(x, lengths) - assert result.shape == (3, 1, 4, 8) - - # Test with different sequence lengths - x = torch.randn(15, 8, 16, dtype=torch.float32, device=device) * 0.1 - x = x.to(dtype=dtype) - lengths = torch.tensor([5, 7, 3], device=device) - result = pack_seq_triton(x, lengths) - assert result.shape == (3, 7, 8, 16) - - -def test_pack_seq_different_block_sizes_fp8(): - """Test pack_seq_triton with different block sizes for fp8.""" - device = "cuda" - dtype = torch.float8_e4m3fn - N, H, D, B = 100, 16, 32, 4 - lengths = torch.tensor([25, 25, 25, 25], device=device) - - x = torch.randn(N, H, D, dtype=torch.float32, device=device) * 0.1 - x = x.to(dtype=dtype) - - # Test different block sizes - for block_t, block_d in [(32, 32), (64, 64), (128, 128)]: - result = pack_seq_triton(x, lengths, block_t=block_t, block_d=block_d) - - assert result.shape == (B, 25, H, D) - - # Check that valid data is preserved (within fp8 precision) - for b in range(B): - start_idx = b * 25 - expected_data = x[start_idx : start_idx + 25].to(torch.float32) - actual_data = result[b, :25].to(torch.float32) - assert_close(actual_data, expected_data, rtol=1e-1, atol=1e-2) - - -def test_pack_seq_shape_consistency(): - """Test that pack_seq_triton maintains shape consistency.""" - device = "cuda" - dtype = torch.float8_e4m3fn - N, H, D, B = 20, 8, 16, 2 - lengths = torch.tensor([10, 10], device=device) - - x = torch.randn(N, H, D, dtype=torch.float32, device=device) * 0.1 - x = x.to(dtype=dtype) - - result = pack_seq_triton(x, lengths) - - # Check shape consistency - assert result.shape[0] == B # Batch dimension - assert result.shape[1] == lengths.max().item() # Max sequence length - assert result.shape[2:] == x.shape[1:] # Feature dimensions preserved - - -def test_pack_unpack_roundtrip_fp8(): - """Test that pack -> unpack gives us back the original data for fp8.""" - device = "cuda" - dtype = torch.float8_e4m3fn - - # Test cases with 3D tensors - test_cases = [ - (6, 8, 4, 2, [3, 3]), - (10, 4, 8, 3, [2, 4, 4]), - (20, 16, 32, 4, [5, 5, 5, 5]), - (15, 8, 16, 3, [7, 5, 3]), - ] - - for N, H, D, B, lengths_list in test_cases: - # Create input tensor with small values for fp8 - x = torch.randn(N, H, D, dtype=torch.float32, device=device) * 0.1 - x = x.to(dtype=dtype) - lengths = torch.tensor(lengths_list, device=device) - - # Pack the data - packed = pack_seq_triton(x, lengths) - - # Unpack the data - unpacked = unpack_seq_triton(packed, lengths) - - # Check that we get back the original data (within fp8 precision) - assert unpacked.shape == x.shape - x_f32 = x.to(torch.float32) - unpacked_f32 = unpacked.to(torch.float32) - assert_close(x_f32, unpacked_f32, rtol=1e-3, atol=1e-3) - - # Unpack without explicit start locations (computed in kernel) - unpacked_with_loc = unpack_seq_triton(packed, lengths) - assert_close(x_f32, unpacked_with_loc.to(torch.float32), rtol=1e-3, atol=1e-2) - - -def test_unpack_seq_triton_edge_cases_fp8(): - """Test unpack function with edge cases for fp8.""" - device = "cuda" - dtype = torch.float8_e4m3fn - - # Test with single batch element - x = torch.randn(10, 8, 16, dtype=torch.float32, device=device) * 0.1 - x = x.to(dtype=dtype) - lengths = torch.tensor([10], device=device) - packed = pack_seq_triton(x, lengths) - unpacked = unpack_seq_triton(packed, lengths) - assert unpacked.shape == x.shape - assert_close(x.to(torch.float32), unpacked.to(torch.float32), rtol=1e-1, atol=1e-2) - - # Test with very short sequences - x = torch.randn(20, 4, 8, dtype=torch.float32, device=device) * 0.1 - x = x.to(dtype=dtype) - lengths = torch.tensor([1, 1, 1], device=device) - packed = pack_seq_triton(x, lengths) - unpacked = unpack_seq_triton(packed, lengths) - # Only compare the first 3 elements that were actually packed - assert_close( - x[:3].to(torch.float32), unpacked.to(torch.float32), rtol=1e-1, atol=1e-2 - ) - - x = torch.randn(15, 8, 16, dtype=torch.float32, device=device) * 0.1 - x = x.to(dtype=dtype) - lengths = torch.tensor([5, 7, 3], device=device) - packed = pack_seq_triton(x, lengths) - unpacked = unpack_seq_triton(packed, lengths) - assert unpacked.shape == x.shape - assert_close(x.to(torch.float32), unpacked.to(torch.float32), rtol=1e-1, atol=1e-2) diff --git a/tests/models/multimodal/generation/vlm_utils/model_utils.py b/tests/models/multimodal/generation/vlm_utils/model_utils.py index 9acbe2c0d6..c6db475698 100644 --- a/tests/models/multimodal/generation/vlm_utils/model_utils.py +++ b/tests/models/multimodal/generation/vlm_utils/model_utils.py @@ -250,8 +250,8 @@ def deepseekvl2_patch_hf_runner(hf_model: HfRunner) -> HfRunner: return BatchFeature(data=inputs, tensor_type="pt") hf_model.processor = processor - hf_model.model.get_output_embeddings = ( - lambda: hf_model.model.language.model.embed_tokens + hf_model.model.get_output_embeddings = lambda: ( + hf_model.model.language.model.embed_tokens ) return hf_model @@ -353,8 +353,8 @@ def glm4v_patch_hf_runner(hf_model: HfRunner) -> HfRunner: ) hf_model.processor = processor - hf_model.model.get_output_embeddings = ( - lambda: hf_model.model.transformer.output_layer + hf_model.model.get_output_embeddings = lambda: ( + hf_model.model.transformer.output_layer ) return hf_model @@ -442,8 +442,8 @@ def h2ovl_patch_hf_runner(hf_model: HfRunner) -> HfRunner: img_context_token_id = hf_model.tokenizer.convert_tokens_to_ids("") hf_model.model.img_context_token_id = img_context_token_id hf_model.processor = H2OVLProcessor(hf_model) - hf_model.model.get_output_embeddings = ( - lambda: hf_model.model.language_model.get_output_embeddings() + hf_model.model.get_output_embeddings = lambda: ( + hf_model.model.language_model.get_output_embeddings() ) hf_model.model.generate = types.MethodType(_internvl_generate, hf_model.model) return hf_model @@ -678,8 +678,8 @@ def skyworkr1v_patch_hf_runner(hf_model: HfRunner) -> HfRunner: img_context_token_id = hf_model.tokenizer.convert_tokens_to_ids("") hf_model.model.img_context_token_id = img_context_token_id hf_model.processor = SkyworkR1VProcessor(hf_model) - hf_model.model.get_output_embeddings = ( - lambda: hf_model.model.language_model.get_output_embeddings() + hf_model.model.get_output_embeddings = lambda: ( + hf_model.model.language_model.get_output_embeddings() ) hf_model.model.generate = types.MethodType(_internvl_generate, hf_model.model) return hf_model @@ -787,8 +787,8 @@ def internvl_patch_hf_runner(hf_model: HfRunner) -> HfRunner: img_context_token_id = hf_model.tokenizer.convert_tokens_to_ids("") hf_model.model.img_context_token_id = img_context_token_id hf_model.processor = InternVLProcessor(hf_model) - hf_model.model.get_output_embeddings = ( - lambda: hf_model.model.language_model.get_output_embeddings() + hf_model.model.get_output_embeddings = lambda: ( + hf_model.model.language_model.get_output_embeddings() ) hf_model.model.generate = types.MethodType(_internvl_generate, hf_model.model) return hf_model @@ -941,8 +941,8 @@ def molmo_patch_hf_runner(hf_model: HfRunner) -> HfRunner: def ovis_patch_hf_runner(hf_model: HfRunner) -> HfRunner: """Patches and returns an instance of the HfRunner to use for Ovis2.""" - hf_model.model.get_output_embeddings = ( - lambda: hf_model.model.llm.get_output_embeddings() + hf_model.model.get_output_embeddings = lambda: ( + hf_model.model.llm.get_output_embeddings() ) def processor(*args, text="", images=None, **kwargs): @@ -977,8 +977,8 @@ def ovis_patch_hf_runner(hf_model: HfRunner) -> HfRunner: def ovis2_5_patch_hf_runner(hf_model: HfRunner) -> HfRunner: """Patches and returns an instance of the HfRunner to use for Ovis2.""" - hf_model.model.get_output_embeddings = ( - lambda: hf_model.model.llm.get_output_embeddings() + hf_model.model.get_output_embeddings = lambda: ( + hf_model.model.llm.get_output_embeddings() ) def processor(*args, text="", images=None, videos=None, **kwargs): @@ -1209,221 +1209,3 @@ def voxtral_patch_hf_runner(hf_model: "HfRunner") -> "HfRunner": hf_model.get_inputs = patched_get_inputs # type: ignore[method-assign] hf_model.model.generate = patched_generate # type: ignore[method-assign] return hf_model - - -def moondream3_processor(model: str): - from vllm.transformers_utils.processors.moondream3 import Moondream3Processor - - return Moondream3Processor.from_pretrained(model, trust_remote_code=True) - - -def moondream3_patch_hf_runner(hf_model: HfRunner) -> HfRunner: - """Patch HfRunner for Moondream3.""" - moondream_processor = hf_model.processor - - def processor(*args, text="", images=None, **kwargs): - if images is None: - return moondream_processor(text=text, **kwargs) - - images_list = [images] if isinstance(images, Image) else images - return moondream_processor(images=images_list, text=text, **kwargs) - - hf_model.processor = processor - - # Expose the LM head for logprob extraction. - hf_model.model.get_output_embeddings = lambda: hf_model.model.model.text.lm_head - - native_model = hf_model.model.model # MoondreamModel instance - - from torch.nn import functional as F - - from vllm.model_executor.models.moondream3 import reconstruct_from_crops - - # Resolve the placeholder tokens from the tokenizer instead of hard-coding. - image_placeholder_ids = moondream_processor.tokenizer.encode( - "", add_special_tokens=False - ) - - def _normalize_tiling(tilings): - """Extract (h, w) tuple from various tiling container formats.""" - tiling = tilings - if isinstance(tiling, torch.Tensor): - tiling = tuple(tiling.squeeze().tolist()) - elif isinstance(tiling, (list, tuple)): - t0 = tiling[0] - if isinstance(t0, torch.Tensor): - tiling = tuple(t0.tolist()) - elif isinstance(t0, (list, tuple)): - tiling = tuple(t0) - return tiling - - def _encode_vision(pixel_values, tilings): - """Run preprocessed crops through vision encoder + projection.""" - device = native_model.device - dtype = native_model.vision.pos_emb.dtype - config = native_model.config - - pv = pixel_values - while pv.dim() > 4: - pv = pv.squeeze(0) - pv = pv.to(device=device, dtype=dtype) - - features = native_model._vis_enc(pv) - grid_size = config.vision.crop_size // config.vision.enc_patch_size - global_feat = features[0] - - if features.shape[0] > 1 and tilings is not None: - tiling = _normalize_tiling(tilings) - local = features[1:].view(-1, grid_size, grid_size, config.vision.enc_dim) - reconstructed = reconstruct_from_crops( - local, - tiling, - config.vision.overlap_margin, - patch_size=1, - ) - else: - reconstructed = global_feat.view( - grid_size, grid_size, config.vision.enc_dim - ) - - return native_model._vis_proj(global_feat, reconstructed) - - def _find_subsequence(seq, subseq): - """Find start index of subseq in seq, or None.""" - n = len(subseq) - for i in range(len(seq) - n + 1): - if seq[i : i + n] == subseq: - return i - return None - - def _generate( - self, - input_ids=None, - pixel_values=None, - tilings=None, - attention_mask=None, - **kwargs, - ): - max_new_tokens = kwargs.get("max_new_tokens", 128) - return_dict = kwargs.get("return_dict_in_generate", False) - output_hs = kwargs.get("output_hidden_states", False) - - if pixel_values is None: - sequences = input_ids - if return_dict: - return types.SimpleNamespace( - sequences=sequences, - hidden_states=() if output_hs else None, - ) - return sequences - - # Processor may return lists; extract the single element. - if isinstance(pixel_values, (list, tuple)): - pixel_values = pixel_values[0] - if ( - isinstance(tilings, (list, tuple)) - and tilings - and not isinstance(tilings[0], int) - ): - tilings = tilings[0] - - hf_model.model._setup_caches() - native_model.use_flex_decoding = False - - device = native_model.device - config = native_model.config - - with torch.inference_mode(): - for block in native_model.text.blocks: - block.kv_cache.k_cache.zero_() - block.kv_cache.v_cache.zero_() - - img_emb = _encode_vision(pixel_values, tilings) - - bos_emb = F.embedding( - torch.tensor([[config.tokenizer.bos_id]], device=device), - native_model.text.wte, - ) - img_input = torch.cat([bos_emb, img_emb.unsqueeze(0)], dim=1) - prefix_len = img_input.size(1) - - mask = native_model.attn_mask[:, :, :prefix_len, :] - pos_ids = torch.arange(prefix_len, dtype=torch.long, device=device) - native_model._prefill(img_input, mask, pos_ids, None) - - ids = input_ids.squeeze(0).tolist() - img_start = _find_subsequence(ids, image_placeholder_ids) - - if img_start is None: - sequences = input_ids - if return_dict: - return types.SimpleNamespace( - sequences=sequences, - hidden_states=() if output_hs else None, - ) - return sequences - - prompt_tokens = ids[img_start + len(image_placeholder_ids) :] - - if not prompt_tokens: - sequences = input_ids - if return_dict: - return types.SimpleNamespace( - sequences=sequences, - hidden_states=() if output_hs else None, - ) - return sequences - - prompt_tensor = torch.tensor([prompt_tokens], device=device) - prompt_emb = F.embedding(prompt_tensor, native_model.text.wte) - prompt_len = prompt_emb.size(1) - - mask = native_model.attn_mask[:, :, prefix_len : prefix_len + prompt_len, :] - pos_ids = torch.arange( - prefix_len, - prefix_len + prompt_len, - dtype=torch.long, - device=device, - ) - hidden = native_model._prefill(prompt_emb, mask, pos_ids, None) - pos = prefix_len + prompt_len - - hidden_last = native_model.text.post_ln(hidden[:, -1:, :]) - logits = native_model.text.lm_head(hidden_last.squeeze(1)) - - generated = [] - all_hidden_states = [] - # Record the hidden state that predicted each generated token. - prev_hs = hidden_last - for _ in range(max_new_tokens): - next_token = logits.argmax(dim=-1).item() - if next_token == 0: - break - generated.append(next_token) - if output_hs: - all_hidden_states.append((prev_hs,)) - - next_emb = F.embedding( - torch.tensor([[next_token]], device=device), - native_model.text.wte, - ) - mask = native_model.attn_mask[:, :, pos : pos + 1, :] - pos_ids_step = torch.tensor([pos], dtype=torch.long, device=device) - hidden = native_model._prefill(next_emb, mask, pos_ids_step, None) - hidden_last = native_model.text.post_ln(hidden[:, -1:, :]) - prev_hs = hidden_last - logits = native_model.text.lm_head(hidden_last.squeeze(1)) - pos += 1 - - result_ids = ids + generated - sequences = torch.tensor([result_ids], device=device) - - if return_dict: - return types.SimpleNamespace( - sequences=sequences, - hidden_states=tuple(all_hidden_states) if output_hs else None, - ) - return sequences - - hf_model.model.generate = types.MethodType(_generate, hf_model.model) - return hf_model diff --git a/tests/models/qwen4_exp/test_ple.py b/tests/models/qwen4_exp/test_ple.py index 6d418d7ad9..220d14e537 100644 --- a/tests/models/qwen4_exp/test_ple.py +++ b/tests/models/qwen4_exp/test_ple.py @@ -15,7 +15,6 @@ from safetensors import safe_open from safetensors.torch import load, load_file, save_file from torch import nn from torch.nn import functional as F -from transformers import Qwen4ExpTextConfig import vllm.model_executor.layers.vocab_parallel_embedding as embedding_module import vllm.model_executor.parameter as parameter_module @@ -31,10 +30,6 @@ from vllm.model_executor.layers.quantization.modelopt import ( ModelOptNvFp4Config, ) from vllm.model_executor.layers.quantization.online.base import OnlineQuantizationConfig -from vllm.models.qwen4_exp.amd import ple_layer as amd_ple_layer -from vllm.models.qwen4_exp.amd.ple_layer import ( - Qwen4ExpPLELayer as Qwen4ExpPLELayerAMD, -) from vllm.models.qwen4_exp.common.ple import ( PLEShardOverlap, compute_ple_shard_overlap, @@ -51,7 +46,6 @@ from vllm.models.qwen4_exp.nvidia.ngram_embedding import ( Qwen4ExpPLEUnquantizedEmbeddingMethod, ) from vllm.models.qwen4_exp.nvidia.ple_layer import Qwen4ExpPLELayer -from vllm.platforms import current_platform from vllm.utils.torch_utils import weak_ref_tensor from vllm.v1.attention.backends.short_conv_attn import ( PleShortConvAttentionMetadata, @@ -732,10 +726,6 @@ def test_ple_embedding_respects_modelopt_exclusion() -> None: ) - - - - def test_ple_embedding_dtype_overrides_modelopt_exclusion() -> None: prefix = "model.layers.1.ple.ple_embedding.ngram_embedding" quant_config = ModelOptNvFp4Config(exclude_modules=[prefix]) @@ -2289,162 +2279,6 @@ def _make_conv_metadata( return metadata, token_offset -@pytest.mark.skipif(not torch.cuda.is_available(), reason="fused conv needs CUDA") -@pytest.mark.parametrize("state_layout", ["SD", "DS"]) -@pytest.mark.parametrize( - "case", - [ - pytest.param( - _ConvBatchCase(num_decodes=1, channels=512), - id="decode-single", - ), - pytest.param( - _ConvBatchCase( - num_decodes=33, - channels=2048, - kernel_size=5, - dilation=1, - ), - id="decode-padded", - ), - pytest.param( - _ConvBatchCase(prefill_query_lens=(37, 0, 5, 128)), - id="prefill", - ), - pytest.param( - _ConvBatchCase( - prefill_query_lens=(16, 676), - channels=10240, - state_index_stride=4, - include_null_state=False, - ), - id="prefill-strided-state-indices", - ), - pytest.param( - _ConvBatchCase( - spec_query_lens=(1, 4, 4, 0), - num_accepted=(1, 2, 4, 1), - spec_query_len=4, - graph_padding=4, - ), - id="spec-graph-padded", - ), - pytest.param( - _ConvBatchCase( - spec_query_lens=(3, 4), - num_accepted=(2, 4), - num_decodes=1, - prefill_query_lens=(5,), - spec_query_len=4, - ), - id="mixed", - ), - pytest.param( - _ConvBatchCase( - spec_query_lens=(1, 2), - num_accepted=(0, 1), - num_decodes=2, - prefill_query_lens=(0, 8, 3), - channels=640, - kernel_size=3, - dilation=2, - spec_query_len=3, - ), - id="mixed-varied", - ), - ], -) -@pytest.mark.parametrize( - ("layer_cls", "fuse_residual"), - [ - pytest.param(Qwen4ExpPLELayer, True, id="nvidia-fused"), - pytest.param(Qwen4ExpPLELayer, False, id="nvidia-unfused"), - pytest.param(Qwen4ExpPLELayerAMD, True, id="amd-fused"), - ], -) -def test_fused_conv_correctness( - case: _ConvBatchCase, - state_layout: str, - layer_cls: type[nn.Module], - fuse_residual: bool, -) -> None: - device = torch.device("cuda") - metadata, num_real_tokens = _make_conv_metadata(case, device) - if case.state_index_stride > 1: - assert metadata.state_indices_tensor.stride(0) == case.state_index_stride - # layer_cls is type[nn.Module], so layer_cls.__new__ is type.__new__. - module = object.__new__(layer_cls) - nn.Module.__init__(module) - module.conv_state_len = (case.kernel_size - 1) * case.dilation - module.short_conv_dilation = case.dilation - - rng, state_reference, conv_state, weights = _make_conv_case( - device, - seed=num_real_tokens + case.channels, - channels=case.channels, - kernel_size=case.kernel_size, - dilation=case.dilation, - state_layout=state_layout, - spec_query_len=case.spec_query_len, - ) - inputs = torch.randn( - metadata.num_actual_tokens, - case.channels, - device=device, - dtype=torch.bfloat16, - generator=rng, - ) - residual = torch.randn( - inputs.shape, - device=device, - dtype=torch.bfloat16, - generator=rng, - ) - outer_residual = torch.randn( - inputs.shape, - device=device, - dtype=torch.bfloat16, - generator=rng, - ) - null_state = conv_state[NULL_BLOCK_ID].clone() - residual_kernel = residual.clone() - # The unfused kernel must overwrite the buffer without reading its contents. - residual_reference = ( - residual.clone() if fuse_residual else torch.zeros_like(residual) - ) - - module._short_conv_dilated_dispatch( - inputs=inputs, - residual=residual_kernel, - outer_residual=outer_residual if fuse_residual else None, - metadata=metadata, - conv_state=conv_state, - conv_weights=weights, - ) - _short_conv_dilated_dispatch_pytorch( - inputs=inputs, - residual=residual_reference, - metadata=metadata, - conv_state=state_reference, - conv_weights=weights, - conv_state_len=module.conv_state_len, - dilation=module.short_conv_dilation, - ) - if fuse_residual: - residual_reference = outer_residual + residual_reference - - assert torch.equal( - residual_kernel[:num_real_tokens], residual_reference[:num_real_tokens] - ) - assert torch.equal(conv_state, state_reference) - assert torch.equal(conv_state[NULL_BLOCK_ID], null_state) - if case.graph_padding: - assert torch.equal( - residual_kernel[num_real_tokens:], - residual_reference[num_real_tokens:], - ) - - @pytest.mark.skipif(not torch.cuda.is_available(), reason="fused gate needs CUDA") @pytest.mark.parametrize(("num_tokens", "strided_kv"), [(1, False), (64, True)]) def test_fused_gate_correctness(num_tokens: int, strided_kv: bool) -> None: @@ -2509,273 +2343,3 @@ def test_fused_gate_correctness(num_tokens: int, strided_kv: bool) -> None: expected_normed = grouped_norm(expected_gated, norm_conv) assert torch.equal(gated, expected_gated) torch.testing.assert_close(normed, expected_normed, atol=1e-2, rtol=1e-2) - - -def _build_amd_ngram_embedding( - monkeypatch: pytest.MonkeyPatch, - *, - device: str, - cpu_offload: bool, - fp8_checkpoint: bool, -) -> tuple[amd_ple_layer.Qwen4ExpNGramEmbedding, torch.Tensor]: - """Construct and load a small AMD embedding through its checkpoint loader.""" - _mock_etp_group(monkeypatch) - monkeypatch.setattr(embedding_module, "get_tensor_model_parallel_rank", lambda: 0) - monkeypatch.setattr( - embedding_module, "get_tensor_model_parallel_world_size", lambda: 1 - ) - monkeypatch.setattr(parameter_module, "get_tensor_model_parallel_rank", lambda: 0) - monkeypatch.setattr( - parameter_module, "get_tensor_model_parallel_world_size", lambda: 1 - ) - - monkeypatch.setattr( - amd_ple_layer, - "get_current_vllm_config", - lambda: SimpleNamespace(engram_config=SimpleNamespace(cpu_offload=cpu_offload)), - ) - config = Qwen4ExpTextConfig( - ngram_size=3, - heads_per_ngram=1, - ngram_vocab_size_base=5, - make_ngram_vocab_size_divisible_by=8, - split_ngram_parts=2, - ple_embed_dim=6, - eos_token_id=0, - vocab_size=64, - ) - with torch.device(device): - module = amd_ple_layer.Qwen4ExpNGramEmbedding( - config, - embedding_dim=config.ple_embed_dim, - ple_dense_layer_id=0, - max_total_tokens=4, - prefix="test.ple_embedding", - layer_name="test.ple", - quant_config=( - Fp8Config(is_checkpoint_fp8_serialized=True) if fp8_checkpoint else None - ), - params_dtype=torch.bfloat16, - ) - # Only the outer owner is a stub; the embedding constructor and loader run. - layer = Qwen4ExpPLELayerAMD.__new__(Qwen4ExpPLELayerAMD) - nn.Module.__init__(layer) - layer.ple_embedding = module - - monkeypatch.setattr( - amd_ple_layer, - "get_forward_context", - lambda: SimpleNamespace(no_compile_layers={module.layer_name: layer}), - ) - - embedding = module.ngram_embedding - loaded_weight = ( - torch.arange(embedding.org_vocab_size * embedding.embedding_dim) - .reshape(embedding.org_vocab_size, embedding.embedding_dim) - .to(embedding.weight.dtype) - ) - shard_size = (embedding.org_vocab_size + 1) // 2 - weights = [ - (f"ngram_embedding.shard_{i}.weight", shard) - for i, shard in enumerate(loaded_weight.split(shard_size)) - ] - expected_loaded = {"ngram_embedding.weight"} - if fp8_checkpoint: - weights.append(("ngram_embedding.weight_scale", torch.tensor([0.25]))) - expected_loaded.add("ngram_embedding.weight_scale") - assert module.load_weights(weights) == expected_loaded - embedding.quant_method.process_weights_after_loading(embedding) - return module, loaded_weight - - -@pytest.mark.parametrize( - "device,cpu_offload", - [ - pytest.param("cpu", False, id="cpu-resident"), - pytest.param( - "cuda:0", - False, - id="gpu-resident", - marks=pytest.mark.skipif( - not torch.cuda.is_available(), reason="requires CUDA or ROCm" - ), - ), - pytest.param( - "cuda:0", - True, - id="gpu-offloaded", - marks=pytest.mark.skipif( - not torch.cuda.is_available(), reason="requires CUDA or ROCm" - ), - ), - ], -) -def test_amd_fp8_embedding_loads_checkpoint_shards_and_global_scale( - monkeypatch: pytest.MonkeyPatch, - device: str, - cpu_offload: bool, -) -> None: - """AMD checkpoint loading must accept and apply the global FP8 scale.""" - module, loaded_weight = _build_amd_ngram_embedding( - monkeypatch, device=device, cpu_offload=cpu_offload, fp8_checkpoint=True - ) - embedding = module.ngram_embedding - assert embedding.weight.dtype == torch.float8_e4m3fn - assert embedding.weight.device == torch.device("cpu" if cpu_offload else device) - if cpu_offload: - assert embedding.weight.is_pinned() - assert embedding.weight_scale.device == torch.device(device) - torch.testing.assert_close(embedding.weight.float().cpu(), loaded_weight.float()) - dequantized = embedding.dequantize(embedding.weight.to(device), torch.bfloat16) - torch.testing.assert_close( - dequantized.cpu(), loaded_weight.to(torch.bfloat16) * 0.25, rtol=0, atol=0 - ) - - -@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA or ROCm") -def test_amd_pinned_embedding_prefetch_machinery_is_lazy( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """The pinned embedding must not allocate the prefetch stream/buffer in init. - - Paths that never prefetch (AMD uses the synchronous pinned-lookup op) - keep the stream/buffer unallocated for the process lifetime. NVIDIA's - first ``start_prefetch`` — the eager profile run, always before cudagraph - capture — allocates them lazily. - """ - module, _ = _build_amd_ngram_embedding( - monkeypatch, device="cuda:0", cpu_offload=True, fp8_checkpoint=False - ) - embedding = module.ngram_embedding - assert embedding._prefetch_buffer is None - assert embedding._prefetch_stream is None - - -@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA or ROCm") -@pytest.mark.parametrize("fp8_checkpoint", [False, True], ids=["bf16", "fp8"]) -def test_amd_pinned_embedding_output_written_under_compile( - monkeypatch: pytest.MonkeyPatch, - fp8_checkpoint: bool, -) -> None: - """The AMD pinned lookup op must survive aot_autograd/Inductor. - - The op writes via the declared-mutated ``output``. Without - ``mutates_args=["output"]`` the call is dead-code-eliminated, the UVA lookup - never runs, and the returned ``output`` stays untouched. - """ - module, loaded_weight = _build_amd_ngram_embedding( - monkeypatch, device="cuda:0", cpu_offload=True, fp8_checkpoint=fp8_checkpoint - ) - - ngram_ids = torch.tensor([[3, 0], [1, 2]], device="cuda:0") - output = torch.zeros( - ngram_ids.shape[0], - module.embedding_dim, - dtype=module.ngram_embedding.weight.dtype, - device="cuda:0", - ) - - def lookup(ids: torch.Tensor, out: torch.Tensor) -> torch.Tensor: - torch.ops.vllm.qwen4_exp_amd_ple_ngram_embedding_pinned( - ids, out, module.layer_name - ) - return out - - torch.compile(lookup, fullgraph=True)(ngram_ids, output) - torch.cuda.current_stream().synchronize() - - expected = loaded_weight[ngram_ids.cpu()].to(device="cuda:0").flatten(-2) - torch.testing.assert_close(output.float(), expected.float(), rtol=0, atol=0) - - -@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA or ROCm") -@pytest.mark.parametrize("fp8_checkpoint", [False, True], ids=["bf16", "fp8"]) -def test_amd_pinned_embedding_output_written_under_cudagraph_capture( - monkeypatch: pytest.MonkeyPatch, - fp8_checkpoint: bool, -) -> None: - """The AMD pinned lookup op must be capture-safe. - - It launches a single kernel on the current stream, so a cudagraph capture - reproduces the lookup into ``output`` without cross-stream work. - """ - module, loaded_weight = _build_amd_ngram_embedding( - monkeypatch, device="cuda:0", cpu_offload=True, fp8_checkpoint=fp8_checkpoint - ) - - ngram_ids = torch.tensor([[3, 0], [1, 2]], device="cuda:0") - output = torch.zeros( - ngram_ids.shape[0], - module.embedding_dim, - dtype=module.ngram_embedding.weight.dtype, - device="cuda:0", - ) - - def lookup(ids: torch.Tensor, out: torch.Tensor) -> torch.Tensor: - torch.ops.vllm.qwen4_exp_amd_ple_ngram_embedding_pinned( - ids, out, module.layer_name - ) - return out - - graph = torch.cuda.CUDAGraph() - warmup_stream = torch.cuda.Stream(device="cuda:0") - warmup_stream.wait_stream(torch.cuda.current_stream()) - with torch.cuda.stream(warmup_stream): - for _ in range(3): - lookup(ngram_ids, output) - torch.cuda.current_stream().wait_stream(warmup_stream) - - with torch.cuda.graph(graph): - lookup(ngram_ids, output) - for _ in range(2): - ngram_ids.copy_(ngram_ids.flip(-1) + 3) - output.fill_(float("nan")) - graph.replay() - torch.cuda.current_stream().synchronize() - - expected = loaded_weight[ngram_ids.cpu()].to(device="cuda:0").flatten(-2) - torch.testing.assert_close(output.float(), expected.float(), rtol=0, atol=0) - - -@pytest.mark.skipif( - not current_platform.is_rocm(), - reason="only the ROCm backend runs the n-gram embedding under torch.compile", -) -@pytest.mark.parametrize( - ("query_lens", "contexts"), - [([3, 1], [[5, 6], [0, 7]]), ([1, 0, 2], [[1, 2], [3, 4], [0, 0]])], - ids=["two-requests", "empty-request"], -) -def test_amd_ngram_embedding_matches_reference_under_compile( - monkeypatch: pytest.MonkeyPatch, - query_lens: list[int], - contexts: list[list[int]], -) -> None: - """The AMD n-gram ID op must run under Dynamo with a symbolic request count.""" - module, loaded_weight = _build_amd_ngram_embedding( - monkeypatch, device="cuda:0", cpu_offload=False, fp8_checkpoint=False - ) - device = torch.device("cuda:0") - query_start_loc = torch.tensor( - [0, *accumulate(query_lens)], dtype=torch.int32, device=device - ) - input_ids = torch.tensor( - [9, 0, 11, 12][: sum(query_lens)], dtype=torch.int32, device=device - ) - ngram_context = torch.tensor(contexts, dtype=torch.int32, device=device) - - compiled = torch.compile(module, fullgraph=True, dynamic=True) - output = compiled(input_ids, query_start_loc, ngram_context) - - ngram_ids = _reference_ngram_ids( - input_ids, - query_start_loc, - ngram_context, - module.layer_multipliers, - module.ngram_heads_vocab_sizes, - module.ngram_heads_offsets, - module.eos_token_id, - module.heads_per_ngram, - ) - expected = loaded_weight[ngram_ids.cpu()].flatten(-2).to(device) - torch.testing.assert_close(output.float(), expected.float(), rtol=0, atol=0) diff --git a/tests/models/transformers/fusers/test_linear.py b/tests/models/transformers/fusers/test_linear.py deleted file mode 100644 index bcad62765a..0000000000 --- a/tests/models/transformers/fusers/test_linear.py +++ /dev/null @@ -1,1439 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Unit tests for the Transformers modeling backend's linear fusers.""" - -import ast -import inspect -from types import MethodType, SimpleNamespace - -import pytest -import torch -import torch.nn as nn -import torch.nn.functional as F - -from vllm.model_executor.models.transformers.fuser import Fusers, get_fuser, get_fusers -from vllm.model_executor.models.transformers.fusers import ( - GLUFuser, - MergedColumnParallelFuser, - PackedQKVFuser, - QKVFuser, - packed_qkv, - qkv, -) -from vllm.model_executor.models.transformers.fx_utils import ( - bypass_existence_guard, - trace, -) - - -class SiluAndMulStub(nn.Module): - """Stand-in for vLLM's `SiluAndMul` (no vLLM config required).""" - - def forward(self, x: torch.Tensor) -> torch.Tensor: - d = x.shape[-1] // 2 - return F.silu(x[..., :d]) * x[..., d:] - - -class NoDownGLU(nn.Module): - """`act(gate(x)) * up(x)` with no output projection -> `down_name` is None.""" - - def __init__(self, hidden: int = 16, inter: int = 32, bias: bool = False): - super().__init__() - self.gate_proj = nn.Linear(hidden, inter, bias=bias) - self.up_proj = nn.Linear(hidden, inter, bias=bias) - self.act_fn = nn.SiLU() - - def forward(self, x): - return self.act_fn(self.gate_proj(x)) * self.up_proj(x) - - -class GLUMLP(NoDownGLU): - """`down(act(gate(x)) * up(x))` — the canonical HF GLU MLP.""" - - def __init__(self, hidden: int = 16, inter: int = 32, bias: bool = False): - super().__init__(hidden, inter, bias) - self.down_proj = nn.Linear(inter, hidden, bias=bias) - - def forward(self, x): - return self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x)) - - -class ReversedGLUMLP(GLUMLP): - """`up(x) * act(gate(x))` — operands swapped (multiply is commutative).""" - - def forward(self, x): - return self.down_proj(self.up_proj(x) * self.act_fn(self.gate_proj(x))) - - -class NotAnMLP(nn.Module): - """Two linears but no activation*linear multiply -> must not match.""" - - def __init__(self): - super().__init__() - self.fc1 = nn.Linear(8, 8) - self.fc2 = nn.Linear(8, 8) - - def forward(self, x): - return self.fc2(self.fc1(x)) - - -class NotAnActGLUMLP(GLUMLP): - """GLU-shaped, but the "activation" is not a known activation module.""" - - def __init__(self): - super().__init__() - self.act_fn = nn.Dropout() - - -class BroadcastGLU(NoDownGLU): - """A broadcast multiply cannot use an equal-halves AndMul kernel.""" - - def __init__(self): - super().__init__() - self.gate_proj = nn.Linear(16, 1, bias=False) - - -class ExtraProjGLU(GLUMLP): - """A third linear on the same input (e.g. a router) -> gate/up still fuse.""" - - def __init__(self): - super().__init__() - self.router = nn.Linear(16, 4, bias=False) - - def forward(self, x): - gated = self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x)) - return gated, self.router(x) - - -class UntraceableMLP(GLUMLP): - """Data-dependent control flow *before* the GLU -> no match.""" - - def forward(self, x): - if x.sum() > 0: # noqa: SIM108 - intentionally untraceable - return self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x)) - return x - - -class UntraceableTailGLUMLP(GLUMLP): - """Data-dependent control flow *after* the GLU -> still fusable.""" - - def forward(self, x): - y = self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x)) - if y.sum() > torch.inf: # intentionally untraceable - y = y * 0 - return y - - -class FakeAttention(nn.Module): - """HF v5-style attention: shape unpacking, dead KV branch, kwargs interface.""" - - is_causal = True - - def __init__( - self, - hidden: int = 32, - head_dim: int = 8, - heads: int = 4, - kv_heads: int = 4, - bias: bool = False, - layer_idx: int = 0, - ): - super().__init__() - self.config = SimpleNamespace(_attn_implementation="vllm") - self.layer_idx = layer_idx - self.head_dim = head_dim - self.scaling = head_dim**-0.5 - self.q_proj = nn.Linear(hidden, heads * head_dim, bias=bias) - self.k_proj = nn.Linear(hidden, kv_heads * head_dim, bias=bias) - self.v_proj = nn.Linear(hidden, kv_heads * head_dim, bias=bias) - self.o_proj = nn.Linear(heads * head_dim, hidden, bias=bias) - - def forward( - self, hidden_states, attention_mask=None, past_key_values=None, **kwargs - ): - from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS - - input_shape = hidden_states.shape[:-1] - hidden_shape = (*input_shape, -1, self.head_dim) - q = self.q_proj(hidden_states).view(hidden_shape).transpose(1, 2) - k = self.k_proj(hidden_states).view(hidden_shape).transpose(1, 2) - v = self.v_proj(hidden_states).view(hidden_shape).transpose(1, 2) - if past_key_values is not None: - k, v = past_key_values.update(k, v, self.layer_idx) - attention_interface = ALL_ATTENTION_FUNCTIONS.get_interface( - self.config._attn_implementation, None - ) - attn_output, attn_weights = attention_interface( - self, q, k, v, attention_mask, scaling=self.scaling, **kwargs - ) - attn_output = attn_output.reshape(*input_shape, -1).contiguous() - return self.o_proj(attn_output), attn_weights - - -class ReversedFakeAttention(FakeAttention): - """Projections computed in (v, k, q) order — q must still be identified.""" - - def forward( - self, hidden_states, attention_mask=None, past_key_values=None, **kwargs - ): - from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS - - input_shape = hidden_states.shape[:-1] - hidden_shape = (*input_shape, -1, self.head_dim) - v = self.v_proj(hidden_states).view(hidden_shape).transpose(1, 2) - k = self.k_proj(hidden_states).view(hidden_shape).transpose(1, 2) - q = self.q_proj(hidden_states).view(hidden_shape).transpose(1, 2) - attention_interface = ALL_ATTENTION_FUNCTIONS.get_interface( - self.config._attn_implementation, None - ) - attn_output, _ = attention_interface( - self, q, k, v, attention_mask, scaling=self.scaling, **kwargs - ) - return self.o_proj(attn_output.reshape(*input_shape, -1)), None - - -class FourParallelLinears(nn.Module): - """Four same-input projections of different widths -> one merged linear.""" - - def __init__(self): - super().__init__() - self.proj_a = nn.Linear(8, 4) - self.proj_b = nn.Linear(8, 6) - self.proj_c = nn.Linear(8, 8) - self.proj_d = nn.Linear(8, 10) - - def forward(self, x): - return self.proj_a(x), self.proj_b(x), self.proj_c(x), self.proj_d(x) - - -class GuardedParallelLinears(FourParallelLinears): - """A projection referenced by an existence guard as well as its call.""" - - def forward(self, x): - d = self.proj_d(x) if self.proj_d is not None else None - return self.proj_a(x), self.proj_b(x), self.proj_c(x), d - - -class MutatingParallelLinears(FourParallelLinears): - def forward(self, x): - a = self.proj_a(x) - x.add_(1) - return a, self.proj_b(x), self.proj_c(x), self.proj_d(x) - - -class AliasMutatingParallelLinears(FourParallelLinears): - """A view of the input is mutated between the projections.""" - - def forward(self, x): - a = self.proj_a(x) - alias = x.view(-1) - alias.mul_(2) - return a, self.proj_b(x), self.proj_c(x), self.proj_d(x) - - -class FunctionalInplaceParallelLinears(FourParallelLinears): - """The input is mutated by a functional in-place call between projections.""" - - def forward(self, x): - a = self.proj_a(x) - F.relu(x, inplace=True) - return a, self.proj_b(x), self.proj_c(x), self.proj_d(x) - - -class UnderscoreInplaceParallelLinears(FourParallelLinears): - """The input is mutated by a free in-place function between projections.""" - - def forward(self, x): - a = self.proj_a(x) - torch.relu_(x) - return a, self.proj_b(x), self.proj_c(x), self.proj_d(x) - - -class PlainQKV(nn.Module): - """q/k/v returned as a bare tuple: no reshape stands between the calls and - the return, so both QKVFuser and MergedColumnParallelFuser can rewrite it.""" - - def __init__(self, hidden: int = 32, heads: int = 4, head_dim: int = 8): - super().__init__() - self.q_proj = nn.Linear(hidden, heads * head_dim, bias=False) - self.k_proj = nn.Linear(hidden, heads * head_dim, bias=False) - self.v_proj = nn.Linear(hidden, heads * head_dim, bias=False) - self.head_dim = head_dim - - def forward(self, x): - return self.q_proj(x), self.k_proj(x), self.v_proj(x) - - -class ExtraProjAttention(FakeAttention): - """A second non-qkv linear of a different width -> `o_proj` still found.""" - - def __init__(self, **kwargs): - super().__init__(**kwargs) - self.sink_proj = nn.Linear(self.head_dim, self.head_dim, bias=False) - - -class QKNormAttention(FakeAttention): - """OLMoE-style: a full-dim norm applied to the whole q/k projection output.""" - - def __init__(self, **kwargs): - super().__init__(**kwargs) - self.q_norm = nn.RMSNorm(self.q_proj.out_features) - self.k_norm = nn.RMSNorm(self.k_proj.out_features) - - def forward( - self, hidden_states, attention_mask=None, past_key_values=None, **kwargs - ): - q = self.q_norm(self.q_proj(hidden_states)) - k = self.k_norm(self.k_proj(hidden_states)) - v = self.v_proj(hidden_states) - return self.o_proj(q + k + v), None - - -class PerHeadQKNormAttention(FakeAttention): - """Qwen3-style: a per-head norm (`head_dim`) applied after the head reshape.""" - - def __init__(self, **kwargs): - super().__init__(**kwargs) - self.q_norm = nn.RMSNorm(self.head_dim) - self.k_norm = nn.RMSNorm(self.head_dim) - - def forward( - self, hidden_states, attention_mask=None, past_key_values=None, **kwargs - ): - shape = (*hidden_states.shape[:-1], -1, self.head_dim) - q = self.q_norm(self.q_proj(hidden_states).view(shape)) - k = self.k_norm(self.k_proj(hidden_states).view(shape)) - v = self.v_proj(hidden_states).view(shape) - return self.o_proj((q + k + v).flatten(-2)), None - - -class ResidDropoutAttention(FakeAttention): - """GPT-style dropout after `o_proj` -> the output projection is still found.""" - - def __init__(self, **kwargs): - super().__init__(**kwargs) - self.resid_dropout = nn.Dropout(0.0) - - def forward( - self, hidden_states, attention_mask=None, past_key_values=None, **kwargs - ): - from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS - - input_shape = hidden_states.shape[:-1] - hidden_shape = (*input_shape, -1, self.head_dim) - q = self.q_proj(hidden_states).view(hidden_shape).transpose(1, 2) - k = self.k_proj(hidden_states).view(hidden_shape).transpose(1, 2) - v = self.v_proj(hidden_states).view(hidden_shape).transpose(1, 2) - attention_interface = ALL_ATTENTION_FUNCTIONS.get_interface( - self.config._attn_implementation, None - ) - attn_output, _ = attention_interface( - self, q, k, v, attention_mask, scaling=self.scaling, **kwargs - ) - attn_output = attn_output.reshape(*input_shape, -1).contiguous() - return self.resid_dropout(self.o_proj(attn_output)), None - - -class GuardedVAttention(FakeAttention): - """Gemma 4-style: `v_proj` is called once but also read in a `None` guard. - - The guard is a fusion invariant (the fuser only binds instances where the - projection exists), so it folds to `True` and the call still fuses.""" - - def forward( - self, hidden_states, attention_mask=None, past_key_values=None, **kwargs - ): - from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS - - input_shape = hidden_states.shape[:-1] - hidden_shape = (*input_shape, -1, self.head_dim) - q = self.q_proj(hidden_states).view(hidden_shape).transpose(1, 2) - k = self.k_proj(hidden_states).view(hidden_shape).transpose(1, 2) - v = ( - self.v_proj(hidden_states).view(hidden_shape).transpose(1, 2) - if self.v_proj is not None - else k - ) - attention_interface = ALL_ATTENTION_FUNCTIONS.get_interface( - self.config._attn_implementation, None - ) - attn_output, _ = attention_interface( - self, q, k, v, attention_mask, scaling=self.scaling, **kwargs - ) - return self.o_proj(attn_output.reshape(*input_shape, -1).contiguous()), None - - -class TruthyGuardAttention(FakeAttention): - """`v_proj` guarded by a bare truthiness test instead of `is not None`. - - An `nn.Linear` is always truthy, so the guard folds to `True` and the call - still fuses (the projection is a proven invariant, exactly as for a `None` - guard).""" - - def forward( - self, hidden_states, attention_mask=None, past_key_values=None, **kwargs - ): - from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS - - input_shape = hidden_states.shape[:-1] - hidden_shape = (*input_shape, -1, self.head_dim) - q = self.q_proj(hidden_states).view(hidden_shape).transpose(1, 2) - k = self.k_proj(hidden_states).view(hidden_shape).transpose(1, 2) - v = ( - self.v_proj(hidden_states).view(hidden_shape).transpose(1, 2) - if self.v_proj - else k - ) - attention_interface = ALL_ATTENTION_FUNCTIONS.get_interface( - self.config._attn_implementation, None - ) - attn_output, _ = attention_interface( - self, q, k, v, attention_mask, scaling=self.scaling, **kwargs - ) - return self.o_proj(attn_output.reshape(*input_shape, -1).contiguous()), None - - -class StatementGuardVAttention(FakeAttention): - """`v_proj` guarded by a statement-form `if self.v_proj:` block. - - The `if`'s test is not an `ast.Compare`, so the identity-check scan skips it; - `_in_boolean_context` folds the test to `True` (an `nn.Linear` is always - truthy) and the call inside the now-`if True:` body still fuses, with the - live `else` surviving the rewrite.""" - - def forward( - self, hidden_states, attention_mask=None, past_key_values=None, **kwargs - ): - from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS - - input_shape = hidden_states.shape[:-1] - hidden_shape = (*input_shape, -1, self.head_dim) - q = self.q_proj(hidden_states).view(hidden_shape).transpose(1, 2) - k = self.k_proj(hidden_states).view(hidden_shape).transpose(1, 2) - if self.v_proj: - v = self.v_proj(hidden_states).view(hidden_shape).transpose(1, 2) - else: - v = k - attention_interface = ALL_ATTENTION_FUNCTIONS.get_interface( - self.config._attn_implementation, None - ) - attn_output, _ = attention_interface( - self, q, k, v, attention_mask, scaling=self.scaling, **kwargs - ) - return self.o_proj(attn_output.reshape(*input_shape, -1).contiguous()), None - - -class BranchedKVAttention(FakeAttention): - """Gemma 4-style: q at body level, k/v inside an untaken `else` branch. - - The three calls do not share a block, so the fused GEMM must be hoisted to - the innermost block that dominates all of them (the function body). The - live `if`/`else` must survive the rewrite.""" - - is_kv_shared_layer = False - - def forward( - self, hidden_states, attention_mask=None, past_key_values=None, **kwargs - ): - from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS - - input_shape = hidden_states.shape[:-1] - hidden_shape = (*input_shape, -1, self.head_dim) - q = self.q_proj(hidden_states).view(hidden_shape).transpose(1, 2) - if self.is_kv_shared_layer: - k = v = None - else: - k = self.k_proj(hidden_states).view(hidden_shape).transpose(1, 2) - v = self.v_proj(hidden_states).view(hidden_shape).transpose(1, 2) - attention_interface = ALL_ATTENTION_FUNCTIONS.get_interface( - self.config._attn_implementation, None - ) - attn_output, _ = attention_interface( - self, q, k, v, attention_mask, scaling=self.scaling, **kwargs - ) - return self.o_proj(attn_output.reshape(*input_shape, -1).contiguous()), None - - -class RebindArgAttention(FakeAttention): - """The projection input is rebound (in an untaken branch) before k/v. - - The branch is dead at trace time so the match succeeds, but the source - rewrite must refuse: hoisting the GEMM above the rebind would feed k/v a - different input. Result: no fusion.""" - - recompute = False - - def forward( - self, hidden_states, attention_mask=None, past_key_values=None, **kwargs - ): - from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS - - input_shape = hidden_states.shape[:-1] - hidden_shape = (*input_shape, -1, self.head_dim) - q = self.q_proj(hidden_states).view(hidden_shape).transpose(1, 2) - if self.recompute: - hidden_states = hidden_states * 2 - k = self.k_proj(hidden_states).view(hidden_shape).transpose(1, 2) - v = self.v_proj(hidden_states).view(hidden_shape).transpose(1, 2) - attention_interface = ALL_ATTENTION_FUNCTIONS.get_interface( - self.config._attn_implementation, None - ) - attn_output, _ = attention_interface( - self, q, k, v, attention_mask, scaling=self.scaling, **kwargs - ) - return self.o_proj(attn_output.reshape(*input_shape, -1)), None - - -class InPlaceMutatedArgAttention(FakeAttention): - """The shared input is mutated in place (in an untaken branch) before k/v. - - Like `RebindArgAttention`, but the mutation keeps the name in a `Load` - context (`hidden_states.mul_(2)`), so a bare `Name`-store check would miss it. - The rewrite must still refuse: hoisting the GEMM above the mutation changes - the value k/v see. Result: no fusion.""" - - recompute = False - - def forward( - self, hidden_states, attention_mask=None, past_key_values=None, **kwargs - ): - from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS - - input_shape = hidden_states.shape[:-1] - hidden_shape = (*input_shape, -1, self.head_dim) - q = self.q_proj(hidden_states).view(hidden_shape).transpose(1, 2) - if self.recompute: - hidden_states.mul_(2) - k = self.k_proj(hidden_states).view(hidden_shape).transpose(1, 2) - v = self.v_proj(hidden_states).view(hidden_shape).transpose(1, 2) - attention_interface = ALL_ATTENTION_FUNCTIONS.get_interface( - self.config._attn_implementation, None - ) - attn_output, _ = attention_interface( - self, q, k, v, attention_mask, scaling=self.scaling, **kwargs - ) - return self.o_proj(attn_output.reshape(*input_shape, -1)), None - - -class AliasMutatedArgAttention(FakeAttention): - """A view of the shared input is mutated before k/v. - - `hidden_states.view(-1)` shares storage with its base, so `alias.mul_(2)` - changes what k/v read without ever naming `hidden_states` as a target. Only - tracking names that may alias the input catches it. Result: no fusion.""" - - recompute = False - - def forward( - self, hidden_states, attention_mask=None, past_key_values=None, **kwargs - ): - from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS - - input_shape = hidden_states.shape[:-1] - hidden_shape = (*input_shape, -1, self.head_dim) - q = self.q_proj(hidden_states).view(hidden_shape).transpose(1, 2) - if self.recompute: - alias = hidden_states.view(-1) - alias.mul_(2) - k = self.k_proj(hidden_states).view(hidden_shape).transpose(1, 2) - v = self.v_proj(hidden_states).view(hidden_shape).transpose(1, 2) - attention_interface = ALL_ATTENTION_FUNCTIONS.get_interface( - self.config._attn_implementation, None - ) - attn_output, _ = attention_interface( - self, q, k, v, attention_mask, scaling=self.scaling, **kwargs - ) - return self.o_proj(attn_output.reshape(*input_shape, -1)), None - - -class FunctionalInplaceArgAttention(FakeAttention): - """The shared input is mutated by a functional in-place call before k/v. - - `F.relu(hidden_states, inplace=True)` writes through an argument, so the - name appears only in a `Load` context inside a call that is not a method on - it -- invisible to a trailing-underscore method check. Result: no fusion.""" - - recompute = False - - def forward( - self, hidden_states, attention_mask=None, past_key_values=None, **kwargs - ): - from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS - - input_shape = hidden_states.shape[:-1] - hidden_shape = (*input_shape, -1, self.head_dim) - q = self.q_proj(hidden_states).view(hidden_shape).transpose(1, 2) - if self.recompute: - F.relu(hidden_states, inplace=True) - k = self.k_proj(hidden_states).view(hidden_shape).transpose(1, 2) - v = self.v_proj(hidden_states).view(hidden_shape).transpose(1, 2) - attention_interface = ALL_ATTENTION_FUNCTIONS.get_interface( - self.config._attn_implementation, None - ) - attn_output, _ = attention_interface( - self, q, k, v, attention_mask, scaling=self.scaling, **kwargs - ) - return self.o_proj(attn_output.reshape(*input_shape, -1)), None - - -class SubscriptMutatedArgAttention(FakeAttention): - """The shared input is written through a subscript (in an untaken branch). - - `hidden_states[..., 0] = 0` leaves the name in a `Load` context on the - subscript's value, so it too evades a bare `Name`-store check. The rewrite - must refuse for the same reason. Result: no fusion.""" - - recompute = False - - def forward( - self, hidden_states, attention_mask=None, past_key_values=None, **kwargs - ): - from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS - - input_shape = hidden_states.shape[:-1] - hidden_shape = (*input_shape, -1, self.head_dim) - q = self.q_proj(hidden_states).view(hidden_shape).transpose(1, 2) - if self.recompute: - hidden_states[..., 0] = 0 - k = self.k_proj(hidden_states).view(hidden_shape).transpose(1, 2) - v = self.v_proj(hidden_states).view(hidden_shape).transpose(1, 2) - attention_interface = ALL_ATTENTION_FUNCTIONS.get_interface( - self.config._attn_implementation, None - ) - attn_output, _ = attention_interface( - self, q, k, v, attention_mask, scaling=self.scaling, **kwargs - ) - return self.o_proj(attn_output.reshape(*input_shape, -1)), None - - -class NonFoldableRefAttention(FakeAttention): - """A surviving `v_proj` reference that is not an existence guard -> no fusion. - - Deleting `v_proj` would break `isinstance(self.v_proj, nn.Linear)`: it reads - the projection's type, not just whether it exists, so the fuser cannot fold - it and must refuse rather than rewrite it.""" - - def forward( - self, hidden_states, attention_mask=None, past_key_values=None, **kwargs - ): - from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS - - input_shape = hidden_states.shape[:-1] - hidden_shape = (*input_shape, -1, self.head_dim) - q = self.q_proj(hidden_states).view(hidden_shape).transpose(1, 2) - k = self.k_proj(hidden_states).view(hidden_shape).transpose(1, 2) - v = self.v_proj(hidden_states).view(hidden_shape).transpose(1, 2) - if isinstance(self.v_proj, nn.Linear): - v = v.contiguous() - attention_interface = ALL_ATTENTION_FUNCTIONS.get_interface( - self.config._attn_implementation, None - ) - attn_output, _ = attention_interface( - self, q, k, v, attention_mask, scaling=self.scaling, **kwargs - ) - return self.o_proj(attn_output.reshape(*input_shape, -1)), None - - -class PackedQKVAttention(nn.Module): - """GPTBigCode-style: one packed projection split into q/k/v in the forward.""" - - is_causal = True - - def __init__( - self, - hidden: int = 32, - head_dim: int = 8, - heads: int = 4, - kv_heads: int = 1, - bias: bool = False, - layer_idx: int = 0, - ): - super().__init__() - self.config = SimpleNamespace(_attn_implementation="vllm") - self.layer_idx = layer_idx - self.head_dim = head_dim - self.scaling = head_dim**-0.5 - self.embed_dim = heads * head_dim - self.kv_dim = kv_heads * head_dim - self.c_attn = nn.Linear(hidden, self.embed_dim + 2 * self.kv_dim, bias=bias) - self.c_proj = nn.Linear(self.embed_dim, hidden, bias=bias) - self.resid_dropout = nn.Dropout(0.0) - - def forward( - self, hidden_states, attention_mask=None, past_key_values=None, **kwargs - ): - from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS - - input_shape = hidden_states.shape[:-1] - q, k, v = ( - self.c_attn(hidden_states) - .unsqueeze(1) - .split((self.embed_dim, self.kv_dim, self.kv_dim), dim=3) - ) - q = q.view(*input_shape, -1, self.head_dim).transpose(1, 2) - if past_key_values is not None: - k, v = past_key_values.update(k, v, self.layer_idx) - attention_interface = ALL_ATTENTION_FUNCTIONS.get_interface( - self.config._attn_implementation, None - ) - attn_output, attn_weights = attention_interface( - self, q, k, v, attention_mask, scaling=self.scaling, **kwargs - ) - attn_output = attn_output.reshape(*input_shape, -1).contiguous() - return self.resid_dropout(self.c_proj(attn_output)), attn_weights - - -class PerHeadSplitAttention(nn.Module): - """A packed projection reshaped and split *per head* -> not a q/k/v split.""" - - def __init__(self, hidden: int = 32, head_dim: int = 8, heads: int = 4): - super().__init__() - self.head_dim = head_dim - self.heads = heads - self.c_attn = nn.Linear(hidden, 3 * heads * head_dim) - self.c_proj = nn.Linear(heads * head_dim, hidden) - - def forward(self, hidden_states): - shape = (*hidden_states.shape[:2], self.heads, 3 * self.head_dim) - q, k, v = ( - self.c_attn(hidden_states) - .view(shape) - .transpose(1, 2) - .split((self.head_dim, self.head_dim, self.head_dim), dim=3) - ) - return self.c_proj((q + k + v).transpose(1, 2).flatten(-2)) - - -class FakeSelfAttn(nn.Module): - """Stand-in for the vLLM `Attention` attached to the dispatching module.""" - - def __init__(self): - super().__init__() - self.impl = SimpleNamespace(scale=None) - - def forward(self, q, k, v): - # MHA-shaped stub: any deterministic combination of q/k/v will do - return q + 2 * k + 3 * v - - -class FakeMQASelfAttn(FakeSelfAttn): - """Stand-in for grouped/multi-query layouts, where `k`/`v` are narrower.""" - - def forward(self, q, k, v): - groups = q.shape[-1] // k.shape[-1] - return q + (2 * k + 3 * v).repeat(1, groups) - - -@pytest.fixture(autouse=True) -def _clear_fuser_cache(): - get_fusers.cache_clear() - yield - get_fusers.cache_clear() - - -def _apply_glu_fuser_with_stubs(module: nn.Module, fuser: GLUFuser): - """Apply a fuser using plain stand-ins (merged `nn.Linear` + silu AndMul).""" - gate = module.get_submodule(fuser.gate_name) - up = module.get_submodule(fuser.up_name) - merged = nn.Linear( - gate.in_features, - gate.out_features + up.out_features, - bias=gate.bias is not None, - ) - with torch.no_grad(): - merged.weight.copy_(torch.cat([gate.weight, up.weight], dim=0)) - if gate.bias is not None: - merged.bias.copy_(torch.cat([gate.bias, up.bias], dim=0)) - setattr(module, fuser.merged_name, merged) - setattr(module, fuser.act_name, SiluAndMulStub()) - delattr(module, fuser.gate_name) - delattr(module, fuser.up_name) - module.forward = MethodType(fuser.fused_forward, module) - return module - - -def _apply_qkv_fuser_with_stubs(module: nn.Module, fuser: QKVFuser): - """Apply a fuser using a plain merged `nn.Linear` (no TP sharding).""" - q, k, v = ( - module.get_submodule(name) - for name in (fuser.q_name, fuser.k_name, fuser.v_name) - ) - merged = nn.Linear( - q.in_features, - q.out_features + k.out_features + v.out_features, - bias=q.bias is not None, - ) - with torch.no_grad(): - merged.weight.copy_(torch.cat([q.weight, k.weight, v.weight], dim=0)) - if q.bias is not None: - merged.bias.copy_(torch.cat([q.bias, k.bias, v.bias], dim=0)) - merged.output_sizes = [q.out_features, k.out_features, v.out_features] - merged.tp_size = 1 - setattr(module, fuser.merged_name, merged) - for name in (fuser.q_name, fuser.k_name, fuser.v_name): - delattr(module, name) - module.forward = MethodType(fuser.fused_forward, module) - return module - - -def _apply_packed_qkv_fuser_with_stubs(module: nn.Module, fuser: PackedQKVFuser): - """Apply a fuser at `tp_size == 1`, where the rewritten split is unchanged.""" - qkv = module.get_submodule(fuser.qkv_name) - qkv.output_sizes = [fuser.q_size, fuser.kv_size, fuser.kv_size] - qkv.tp_size = 1 - module.forward = MethodType(fuser.fused_forward, module) - return module - - -@pytest.mark.parametrize("mlp_cls", [GLUMLP, ReversedGLUMLP]) -@pytest.mark.parametrize("bias", [False, True]) -def test_detects_and_rewrites_glu(mlp_cls, bias): - with torch.device("meta"): - meta = mlp_cls(bias=bias) - fuser = get_fuser(meta, GLUFuser) - assert isinstance(fuser, GLUFuser) - assert ( - fuser.gate_name, - fuser.up_name, - fuser.act_name, - fuser.down_name, - ) == ("gate_proj", "up_proj", "act_fn", "down_proj") - - # The rewritten forward references the merged projection instead of the - # sources; the rest of the forward is untouched. - names = fuser.fused_forward.__code__.co_names - assert "gate_up_proj" in names and "act_fn" in names and "down_proj" in names - assert not {"gate_proj", "up_proj"} & set(names) - - # Numerics: the fused forward must match the original on a real instance. - real = mlp_cls(bias=bias) - for p in real.parameters(): - nn.init.normal_(p, std=0.05) - x = torch.randn(4, 16) - expected = real(x) - fused = _apply_glu_fuser_with_stubs(real, fuser) - - # Fusion is in place: the module keeps its class and other attributes - assert fused is real and type(fused) is mlp_cls - torch.testing.assert_close(fused(x), expected, atol=1e-5, rtol=1e-5) - - -def test_glu_fuses_alongside_unrelated_projections(): - """Sibling linears the GLU does not consume must not block the fusion.""" - with torch.device("meta"): - fuser = get_fuser(ExtraProjGLU(), GLUFuser) - assert fuser is not None - assert (fuser.gate_name, fuser.up_name) == ("gate_proj", "up_proj") - real = ExtraProjGLU() - x = torch.randn(4, 16) - expected = real(x) - fused = _apply_glu_fuser_with_stubs(real, fuser) - torch.testing.assert_close(fused(x), expected) - - -def test_glu_identifies_down_projection(): - """The row projection consuming `act(gate(x)) * up(x)` is identified. - - It is forced to `RowParallelLinear` in `update_attrs` so its sharded input - matches the column-parallel merged gate/up; `None` when there is no such - projection to force (fusion of gate/up still applies).""" - with torch.device("meta"): - assert get_fuser(GLUMLP(), GLUFuser).down_name == "down_proj" - assert get_fuser(ReversedGLUMLP(), GLUFuser).down_name == "down_proj" - assert get_fuser(NoDownGLU(), GLUFuser).down_name is None - - -@pytest.mark.parametrize("attn_cls", [FakeAttention, ReversedFakeAttention]) -@pytest.mark.parametrize("kv_heads", [4, 2]) -def test_detects_and_rewrites_qkv(attn_cls, kv_heads): - if attn_cls is ReversedFakeAttention and kv_heads == 4: - pytest.skip("MHA q/k/v assignment is order-based by design") - with torch.device("meta"): - meta = attn_cls(kv_heads=kv_heads) - fuser = get_fuser(meta, QKVFuser) - assert isinstance(fuser, QKVFuser) - # q (sharded differently under TP) must be identified exactly; k/v may be - # swapped for non-canonical compute order, which is numerically consistent - # because the weight mapping and the split indices follow the same - # assignment. - assert fuser.q_name == "q_proj" - assert {fuser.k_name, fuser.v_name} == {"k_proj", "v_proj"} - assert fuser.o_name == "o_proj" - - # The projections are merged; everything else stays live Python with its - # original semantics (branches, kwargs, attribute reads) - code = fuser.fused_forward.__code__ - names = code.co_names - assert "qkv_proj" in names and "output_sizes" in names and "o_proj" in names - assert "tp_size" in names - assert not {"q_proj", "k_proj", "v_proj"} & set(names) - if attn_cls is FakeAttention: - assert "update" in names # the cache branch survives - assert code.co_flags & inspect.CO_VARKEYWORDS # **kwargs survives - - # Numerics: the fused forward must match the original on a real instance, - # with a different layer_idx than the traced instance (kv_heads == heads so - # the q/k/v stub combination is shape-compatible). - real = attn_cls(kv_heads=4, layer_idx=3) - for p in real.parameters(): - nn.init.normal_(p, std=0.05) - x = torch.randn(1, 5, 32) - real.attn = FakeSelfAttn() - expected, _ = real(x) - fused = _apply_qkv_fuser_with_stubs(real, fuser) - - # Fusion is in place: the module keeps its class and other attributes - assert fused is real and type(fused) is attn_cls - assert fused.layer_idx == 3 and fused.is_causal and fused.config is not None - out, _ = fused(x) - torch.testing.assert_close(out, expected, atol=1e-5, rtol=1e-5) - - -def test_qkv_identifies_output_projection(): - with torch.device("meta"): - assert get_fuser(FakeAttention(), QKVFuser).o_name == "o_proj" - assert get_fuser(ReversedFakeAttention(), QKVFuser).o_name == "o_proj" - assert get_fuser(ExtraProjAttention(), QKVFuser).o_name == "o_proj" - # Norm children (q_norm/k_norm) must not disturb o_proj identification. - assert get_fuser(QKNormAttention(), QKVFuser).o_name == "o_proj" - assert get_fuser(PerHeadQKNormAttention(), QKVFuser).o_name == "o_proj" - # A module between o_proj and the return is transparent. - assert get_fuser(ResidDropoutAttention(), QKVFuser).o_name == "o_proj" - - -@pytest.mark.parametrize( - "attn_cls", - [ - GuardedVAttention, - TruthyGuardAttention, - StatementGuardVAttention, - BranchedKVAttention, - ], -) -def test_fuses_gemma4_qkv_obstacles(attn_cls): - """A guarded `v_proj` (obstacle 1) or branch-split k/v (obstacle 2) fuses. - - Both patterns leave `v_proj` referenced beyond its single call, or split the - calls across blocks; the rewrite folds the guard / hoists the GEMM and the - numerics still match. The guard may be a `None` comparison or a bare - truthiness test (ternary or statement form); all are proven invariants, so - folding them is sound. The naive "relax the reference count" fix fuses the - guarded cases but silently drops the guard's semantics. - """ - with torch.device("meta"): - meta = attn_cls() - fuser = get_fuser(meta, QKVFuser) - assert isinstance(fuser, QKVFuser) - assert (fuser.q_name, fuser.o_name) == ("q_proj", "o_proj") - - names = set(fuser.fused_forward.__code__.co_names) - assert "qkv_proj" in names - assert not {"q_proj", "k_proj", "v_proj"} & names - if attn_cls is BranchedKVAttention: - assert "is_kv_shared_layer" in names # the live branch survives - - real = attn_cls(kv_heads=4, layer_idx=3) - for p in real.parameters(): - nn.init.normal_(p, std=0.05) - x = torch.randn(1, 5, 32) - real.attn = FakeSelfAttn() - expected, _ = real(x) - fused = _apply_qkv_fuser_with_stubs(real, fuser) - out, _ = fused(x) - torch.testing.assert_close(out, expected, atol=1e-5, rtol=1e-5) - - -@pytest.mark.parametrize( - "attn_cls", - [ - RebindArgAttention, - InPlaceMutatedArgAttention, - SubscriptMutatedArgAttention, - AliasMutatedArgAttention, - FunctionalInplaceArgAttention, - NonFoldableRefAttention, - ], -) -def test_qkv_refuses_unsound_rewrites(attn_cls): - """The match succeeds but the source rewrite must refuse, leaving no fuser. - - `RebindArgAttention` rebinds the shared input before k/v; the `*Mutated*` and - `FunctionalInplace*` cases mutate it in place, directly (`x.mul_(...)`, - `x[...] = 0`), through a view that shares its storage, or through a call that - writes an argument (`F.relu(x, inplace=True)`) -- each would change what k/v - see if the GEMM were hoisted above them. - `NonFoldableRefAttention` reads `v_proj` outside an existence guard (the fuser - cannot fold it away). All fail closed in `update_forward`, so `get_fuser` - returns `None`. - """ - with torch.device("meta"): - meta = attn_cls() - assert get_fuser(meta, QKVFuser) is None - - -def _guard_funcdef(expr: str) -> tuple[ast.FunctionDef, ast.Attribute]: - """A `def f(self, a, b, T): return ` and its `self.v_proj` reference.""" - funcdef = ast.parse(f"def f(self, a, b, T):\n return {expr}").body[0] - assert isinstance(funcdef, ast.FunctionDef) - ref = next( - node - for node in ast.walk(funcdef) - if isinstance(node, ast.Attribute) and node.attr == "v_proj" - ) - return funcdef, ref - - -@pytest.mark.parametrize( - "expr, expected", - [ - ("a if self.v_proj is not None else b", True), - ("a if self.v_proj is None else b", False), - ("a if None is not self.v_proj else b", True), # None on the left - ], -) -def test_bypass_existence_guard_folds_identity_none_check(expr, expected): - """`is`/`is not None`, either operand order, fold to a bool. - - The projection is a proven invariant (it exists as an `nn.Linear`), so an - `is (not) None` guard has a constant truth value; folding it lets the call be - rewritten away without changing the guard's outcome. `==`/`!=` are excluded - (see the refusal test) because a subclass may override `__eq__`/`__ne__`.""" - funcdef, ref = _guard_funcdef(expr) - bypass_existence_guard(funcdef, ref, "v_proj") - test = next(node for node in ast.walk(funcdef) if isinstance(node, ast.IfExp)).test - assert isinstance(test, ast.Constant) and test.value is expected - - -@pytest.mark.parametrize( - "expr", - [ - "a if self.v_proj else b", # a ternary test - "not self.v_proj", # a `not` operand - ], -) -def test_bypass_existence_guard_folds_bare_truthiness(expr): - """A bare truthiness test folds the reference itself to `True`. - - An `nn.Linear` has no `__bool__`/`__len__`, so it is always truthy; in a pure - boolean position (a test, or a `not` operand) the reference can be replaced by - `True`, letting the projection be deleted.""" - funcdef, ref = _guard_funcdef(expr) - bypass_existence_guard(funcdef, ref, "v_proj") - # No `self.v_proj` reference survives; a literal `True` took its place. - attrs = [n for n in ast.walk(funcdef) if isinstance(n, ast.Attribute)] - assert not any(n.attr == "v_proj" for n in attrs) - constants = [n for n in ast.walk(funcdef) if isinstance(n, ast.Constant)] - assert any(n.value is True for n in constants) - - -def test_bypass_existence_guard_folds_bare_if_statement(): - """A statement-form `if self.:` folds its test to `True`. - - The `len(node.ops) == 1` `Compare` scan skips it (it is not a comparison at - all); `_in_boolean_context` then matches the `if`'s test and folds it, so the - bare-truthiness statement form is handled, not silently dropped.""" - funcdef = ast.parse( - "def f(self, a, b):\n if self.v_proj:\n return a\n return b" - ).body[0] - assert isinstance(funcdef, ast.FunctionDef) - ref = next( - node - for node in ast.walk(funcdef) - if isinstance(node, ast.Attribute) and node.attr == "v_proj" - ) - bypass_existence_guard(funcdef, ref, "v_proj") - test = next(node for node in ast.walk(funcdef) if isinstance(node, ast.If)).test - assert isinstance(test, ast.Constant) and test.value is True - - -@pytest.mark.parametrize( - "expr", - [ - "a if isinstance(self.v_proj, T) else b", # a different predicate - "self.v_proj.weight", # an attribute read, not a guard - "a and self.v_proj", # `and` yields the operand, which may escape - "self.v_proj or a", # `or` yields the module itself when truthy - "a if self.v_proj == None else b", # `__eq__` may not track identity - "a if self.v_proj != None else b", # `__ne__` may not track identity - ], -) -def test_bypass_existence_guard_refuses_non_guard_reference(expr): - """A surviving reference that is not an identity guard cannot be folded away. - - Rewriting it is unsafe (its value depends on more than the projection's - existence), so the fuser refuses rather than change semantics. `==`/`!=` are - refused too: `is_linear` accepts `nn.Linear` subclasses that could override - `__eq__`/`__ne__`, so equality need not agree with `is (not) None`.""" - funcdef, ref = _guard_funcdef(expr) - with pytest.raises(ValueError, match="outside an existence guard"): - bypass_existence_guard(funcdef, ref, "v_proj") - - -@pytest.mark.parametrize("layer_idx", [0, 1]) -def test_fuses_real_gemma4_attention(layer_idx): - """The real Gemma 4 attention (both obstacles at once) fuses end to end.""" - pytest.importorskip("transformers.models.gemma4") - from transformers.models.gemma4.configuration_gemma4 import Gemma4TextConfig - from transformers.models.gemma4.modeling_gemma4 import Gemma4TextAttention - - config = Gemma4TextConfig( - hidden_size=64, - num_attention_heads=4, - num_key_value_heads=2, - num_hidden_layers=2, - head_dim=16, - intermediate_size=128, - ) - with torch.device("meta"): - meta = Gemma4TextAttention(config, layer_idx=layer_idx) - fuser = get_fuser(meta, QKVFuser) - assert isinstance(fuser, QKVFuser) - assert (fuser.q_name, fuser.k_name, fuser.v_name) == ( - "q_proj", - "k_proj", - "v_proj", - ) - assert fuser.o_name == "o_proj" - names = set(fuser.fused_forward.__code__.co_names) - assert not {"q_proj", "k_proj", "v_proj"} & names - - -def test_merged_column_fuser_folds_existence_guard(): - """A guarded projection fuses: the merged linear replaces the deleted names. - - Guard folding lives on `StackedFuser`, not just `QKVFuser`, because every - stacked fuser deletes the projections it merges -- a surviving reference to - one would fail at runtime.""" - with torch.device("meta"): - module = GuardedParallelLinears() - fuser = MergedColumnParallelFuser.match(trace(module), module) - assert fuser is not None - fuser.update_forward(module) - names = set(fuser.fused_forward.__code__.co_names) - assert fuser.merged_name in names - assert not {"proj_a", "proj_b", "proj_c", "proj_d"} & names - - -@pytest.mark.parametrize( - "cls", - [ - MutatingParallelLinears, - AliasMutatingParallelLinears, - FunctionalInplaceParallelLinears, - UnderscoreInplaceParallelLinears, - ], -) -def test_merged_column_fuser_rejects_input_mutation(cls): - """The later projections must not be moved ahead of an input mutation. - - The mutation may be direct (`x.add_(1)`), through a view that shares the - input's storage, or through a call that writes an argument in place - (`F.relu(x, inplace=True)`, `torch.relu_(x)`). Each one silently changes the - fused result, so the rewrite must refuse rather than fuse.""" - with torch.device("meta"): - module = cls() - fuser = MergedColumnParallelFuser.match(trace(module), module) - assert fuser is not None - with pytest.raises(ValueError, match="rebound or mutated"): - fuser.update_forward(module) - - -@pytest.mark.parametrize("cls", [FourParallelLinears, GuardedParallelLinears]) -def test_merged_column_fuser_supports_any_number_of_linears( - cls, default_vllm_config, monkeypatch -): - from vllm.model_executor import parameter - from vllm.model_executor.layers import linear - from vllm.model_executor.layers.linear import MergedColumnParallelLinear - from vllm.model_executor.layers.utils import default_unquantized_gemm - from vllm.model_executor.models.utils import AutoWeightsLoader, WeightsMapper - - monkeypatch.setattr(linear, "get_tensor_model_parallel_rank", lambda: 0) - monkeypatch.setattr(linear, "get_tensor_model_parallel_world_size", lambda: 1) - monkeypatch.setattr( - linear, - "dispatch_unquantized_gemm", - lambda linear_backend="auto": default_unquantized_gemm, - ) - monkeypatch.setattr(parameter, "get_tensor_model_parallel_rank", lambda: 0) - monkeypatch.setattr(parameter, "get_tensor_model_parallel_world_size", lambda: 1) - with torch.device("meta"): - meta = cls() - fuser = MergedColumnParallelFuser.match(trace(meta), meta) - assert fuser is not None - assert set(fuser.linear_names) == {"proj_a", "proj_b", "proj_c", "proj_d"} - assert fuser.shards == [(name, i) for i, name in enumerate(fuser.linear_names)] - fuser.update_forward(meta) - - real = cls() - x = torch.randn(2, 8) - expected = real(x) - weights = [ - (name, weight.detach().clone()) for name, weight in real.named_parameters() - ] - fused = fuser.fuse(real, "parallel", default_vllm_config) - root = nn.Module() - root.parallel = fused - mapper = WeightsMapper(orig_to_new_stacked=fuser.orig_to_new_stacked("parallel")) - AutoWeightsLoader(root).load_weights( - ((f"parallel.{name}", weight) for name, weight in weights), mapper=mapper - ) - - merged = getattr(fused, fuser.merged_name) - assert isinstance(merged, MergedColumnParallelLinear) - for actual, reference in zip(fused(x), expected): - torch.testing.assert_close(actual, reference) - - -def test_merged_name_is_unique_per_linear_names(): - """The computed name must not collide across different projections, so - stacking two unrelated fusions can never clobber packed_modules_mapping.""" - ab_c = MergedColumnParallelFuser(source_cls="M", linear_names=("a_b", "c")) - a_bc = MergedColumnParallelFuser(source_cls="M", linear_names=("a", "b_c")) - assert ab_c.merged_name != a_bc.merged_name - # Same names -> same (and therefore safely re-mergeable) name. - dup = MergedColumnParallelFuser(source_cls="M", linear_names=("a_b", "c")) - assert ab_c.merged_name == dup.merged_name - - -def test_heterogeneous_instances_fall_back_to_merged_column_fuser( - default_vllm_config, -): - """Same class and structure -> both fusers are cached as candidates. A - layer whose out_features aren't a multiple of its own head_dim (as a - misconfigured or heterogeneous checkpoint might have) fails QKVFuser's - validation and must still fuse via the generic fallback, while a - well-formed sibling of the same class keeps using QKVFuser.""" - with torch.device("meta"): - container = nn.Module() - container.compatible = PlainQKV(head_dim=8) - container.incompatible = PlainQKV(head_dim=8) - container.incompatible.head_dim = 5 # 32 % 5 != 0, unlike q_proj's 32 % 8 - - candidates = get_fusers(container.compatible) - assert any(isinstance(f, QKVFuser) for f in candidates) - assert any(isinstance(f, MergedColumnParallelFuser) for f in candidates) - - fusers = Fusers(container, default_vllm_config) - assert isinstance(fusers[container.compatible][0], QKVFuser) - assert isinstance(fusers[container.incompatible][0], MergedColumnParallelFuser) - - -@pytest.mark.parametrize("kv_heads", [1, 2]) -def test_detects_and_rewrites_packed_qkv(kv_heads): - """A single projection split into q/k/v must be re-sharded, not merged. - - Only the split sizes change: `QKVParallelLinear` loads the packed - checkpoint weight as-is, and shards q by heads while replicating k/v.""" - with torch.device("meta"): - meta = PackedQKVAttention(kv_heads=kv_heads) - fuser = get_fuser(meta, PackedQKVFuser) - assert isinstance(fuser, PackedQKVFuser) - assert (fuser.qkv_name, fuser.o_name) == ("c_attn", "c_proj") - assert (fuser.q_size, fuser.kv_size) == (32, 8 * kv_heads) - - # The hard-coded widths become the per-rank widths of the sharded linear - names = fuser.fused_forward.__code__.co_names - assert "output_sizes" in names and "tp_size" in names - assert "kv_dim" not in names and "embed_dim" not in names - - # Numerics: the rewritten forward must match the original on a real instance - real = PackedQKVAttention(kv_heads=kv_heads, layer_idx=3) - for p in real.parameters(): - nn.init.normal_(p, std=0.05) - x = torch.randn(1, 5, 32) - real.attn = FakeMQASelfAttn() - expected, _ = real(x) - fused = _apply_packed_qkv_fuser_with_stubs(real, fuser) - - # Fusion is in place: the module keeps its class and other attributes - assert fused is real and type(fused) is PackedQKVAttention - assert fused.layer_idx == 3 and fused.is_causal - out, _ = fused(x) - torch.testing.assert_close(out, expected, atol=1e-5, rtol=1e-5) - - -def test_per_head_split_is_not_packed_qkv(): - """The split must consume the whole projection, else its sizes are head - widths and re-sharding by them would be wrong.""" - with torch.device("meta"): - assert get_fuser(PerHeadSplitAttention(), PackedQKVFuser) is None - - -def test_fuser_is_cached_per_class_and_structure(): - with torch.device("meta"): - fuser_a = get_fuser(GLUMLP(), GLUFuser) - fuser_b = get_fuser(GLUMLP(), GLUFuser) - assert fuser_a is fuser_b - assert any(key[0] is GLUMLP for key in get_fusers.cache) - - -@pytest.mark.parametrize("cls", [NotAnMLP, UntraceableMLP]) -def test_non_matching_modules_return_none(cls): - with torch.device("meta"): - module = cls() - assert get_fuser(module, GLUFuser) is None - - -def test_untraceable_tail_still_fuses(): - with torch.device("meta"): - meta = UntraceableTailGLUMLP() - fuser = get_fuser(meta, GLUFuser) - assert isinstance(fuser, GLUFuser) - - # Numerics: the live tail must survive the rewrite - real = UntraceableTailGLUMLP() - for p in real.parameters(): - nn.init.normal_(p, std=0.05) - x = torch.randn(4, 16) - expected = real(x) - fused = _apply_glu_fuser_with_stubs(real, fuser) - torch.testing.assert_close(fused(x), expected, atol=1e-5, rtol=1e-5) - - -def test_weight_mappings_are_scoped_to_fused_prefixes(): - from vllm.model_executor.models.utils import WeightsMapper - - with torch.device("meta"): - glu_fuser = get_fuser(GLUMLP(), GLUFuser) - qkv_fuser = get_fuser(FakeAttention(), QKVFuser) - - mapper = WeightsMapper() - for prefix in ("model.layers.0.mlp", "model.layers.1.mlp"): - mapper.orig_to_new_stacked.update(glu_fuser.orig_to_new_stacked(prefix)) - mapper.orig_to_new_stacked.update( - qkv_fuser.orig_to_new_stacked("model.layers.0.self_attn") - ) - - names = [ - "model.layers.0.mlp.gate_proj.weight", - "model.layers.0.mlp.up_proj.weight", - "model.layers.1.mlp.gate_proj.weight", - "model.layers.0.self_attn.q_proj.weight", - "model.layers.0.self_attn.k_proj.weight", - "model.layers.0.self_attn.v_proj.weight", - # Unfused modules at other prefixes must be left untouched. - "model.layers.2.mlp.experts.0.gate_proj.weight", - "model.layers.1.self_attn.q_proj.weight", - ] - # `apply` rewrites the name and stamps the shard id onto each tensor. - weights = [(name, torch.empty(0)) for name in names] - mapped = list(mapper.apply(weights)) - mapped_names = [name for name, _ in mapped] - shard_ids = [getattr(data, "shard_id", None) for _, data in mapped] - - assert mapped_names == [ - "model.layers.0.mlp.gate_up_proj.weight", - "model.layers.0.mlp.gate_up_proj.weight", - "model.layers.1.mlp.gate_up_proj.weight", - "model.layers.0.self_attn.qkv_proj.weight", - "model.layers.0.self_attn.qkv_proj.weight", - "model.layers.0.self_attn.qkv_proj.weight", - # Only the exact fused layers are remapped; everything else is untouched. - "model.layers.2.mlp.experts.0.gate_proj.weight", - "model.layers.1.self_attn.q_proj.weight", - ] - assert shard_ids == [0, 1, 0, "q", "k", "v", None, None] - - # The fused layers are exposed to the quantization machinery via their - # original constituent projection names (what the checkpoint stores). - assert glu_fuser.packed_modules_mapping == { - "gate_up_proj": ["gate_proj", "up_proj"], - } - assert qkv_fuser.packed_modules_mapping == { - "qkv_proj": ["q_proj", "k_proj", "v_proj"], - } - - -@pytest.mark.parametrize("cls", [NotAnMLP, NotAnActGLUMLP, BroadcastGLU]) -def test_unfusable_modules_are_not_fused(cls, default_vllm_config): - with torch.device("meta"): - module = cls() - fuser = get_fuser(module, GLUFuser) - # Either no pattern matches the class, or this instance fails validation - # (`recursive_replace` gates fusion and its weight mappings on `validate`) - assert fuser is None or not fuser.validate(module, default_vllm_config) - - -def test_act_and_mul_derived_from_module(default_vllm_config): - from transformers.activations import GELUTanh, SiLUActivation - - from vllm.model_executor.layers.activation import GeluAndMul, SiluAndMul - - assert isinstance(GLUFuser._get_act_and_mul(nn.SiLU()), SiluAndMul) - assert isinstance(GLUFuser._get_act_and_mul(SiLUActivation()), SiluAndMul) - gelu_tanh = GLUFuser._get_act_and_mul(GELUTanh()) - assert isinstance(gelu_tanh, GeluAndMul) and gelu_tanh.approximate == "tanh" - gelu = GLUFuser._get_act_and_mul(nn.GELU()) - assert isinstance(gelu, GeluAndMul) and gelu.approximate == "none" - # Not activations at all -> no fusion - assert GLUFuser._get_act_and_mul_name(nn.Dropout()) is None - assert GLUFuser._get_act_and_mul_name(nn.LayerNorm(8)) is None - with pytest.raises(ValueError, match="No AndMul equivalent"): - GLUFuser._get_act_and_mul(nn.Dropout()) - - -def _wider_model_config(head_dim: int) -> SimpleNamespace: - """A model whose global head size is twice `head_dim`, as a wider layer - elsewhere in a heterogeneous checkpoint would make it.""" - return SimpleNamespace( - model_config=SimpleNamespace(get_head_size=lambda: 2 * head_dim), - quant_config=None, - ) - - -@pytest.mark.parametrize( - "cls, fuser_module, fuser_cls", - [ - (FakeAttention, qkv, QKVFuser), - (PackedQKVAttention, packed_qkv, PackedQKVFuser), - ], -) -def test_head_counts_come_from_the_module_not_the_model( - cls, fuser_module, fuser_cls, monkeypatch -): - """A layer narrower than the model-wide head size must not be miscounted. - - On a heterogeneous checkpoint (Gemma 4) the model-wide head size is the - largest across layers, so deriving `total_num_heads = out_features // - head_size` from it undercounts heads on a narrower layer. The widths still - add up, so nothing raises below TP=4: the layer is just sharded wrong. - """ - head_dim, heads, kv_heads = 8, 8, 4 - vllm_config = _wider_model_config(head_dim) - with torch.device("meta"): - module = cls(hidden=32, head_dim=head_dim, heads=heads, kv_heads=kv_heads) - - # Both replacements shard, so they need a TP group; only the head counts - # the fuser derives are under test here. - captured = {} - monkeypatch.setattr( - fuser_module, - "QKVParallelLinear", - lambda **kwargs: captured.update(kwargs) or nn.Identity(), - ) - monkeypatch.setattr( - fuser_module, "replace_linear_class", lambda *a, **kw: nn.Identity() - ) - fuser = get_fuser(module, fuser_cls) - assert fuser is not None and fuser.validate(module, vllm_config) - fuser.update_attrs(module, "model.layers.0.self_attn", vllm_config) - - assert captured["head_size"] == head_dim - assert captured["total_num_heads"] == heads - assert captured["total_num_kv_heads"] == kv_heads - - -def test_validate_accepts_a_layer_the_model_wide_head_size_would_reject(): - """`validate` gates fusion, so a wrong head size silently disables it.""" - head_dim, heads, kv_heads = 8, 8, 3 - vllm_config = _wider_model_config(head_dim) - with torch.device("meta"): - module = FakeAttention( - hidden=32, head_dim=head_dim, heads=heads, kv_heads=kv_heads - ) - - # kv width is 24, not a multiple of the model-wide 16, but is of this - # layer's 8. - fuser = get_fuser(module, QKVFuser) - assert fuser is not None and fuser.validate(module, vllm_config) diff --git a/tests/models/transformers/fusers/test_mla.py b/tests/models/transformers/fusers/test_mla.py deleted file mode 100644 index 625fc4dc08..0000000000 --- a/tests/models/transformers/fusers/test_mla.py +++ /dev/null @@ -1,181 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Unit tests for the Transformers modeling backend's MLA fuser. - -The fuser must discover every MLA submodule structurally (never by assuming the -Transformers attribute names) and, when the query is low-rank, merge the checkpoint's -separate `q_a_proj`/`kv_a_proj_with_mqa` weights into the single fused down-projection. -""" - -import pytest -import torch -import torch.nn as nn -import torch.nn.functional as F - -from vllm.model_executor.models.transformers.fuser import get_fuser -from vllm.model_executor.models.transformers.fusers import MLAFuser -from vllm.model_executor.models.transformers.fx_utils import trace - -_FUSED_QKV_A_PROJ = MLAFuser.merged_name - - -def _attention(q_lora_rank: int | None): - """A meta `DeepseekV2Attention` in the requested q_lora configuration.""" - pytest.importorskip("transformers.models.deepseek_v2.modeling_deepseek_v2") - from transformers.models.deepseek_v2.configuration_deepseek_v2 import ( - DeepseekV2Config, - ) - from transformers.models.deepseek_v2.modeling_deepseek_v2 import ( - DeepseekV2Attention, - ) - - cfg = DeepseekV2Config( - hidden_size=256, - num_attention_heads=16, - kv_lora_rank=128, - qk_rope_head_dim=32, - qk_nope_head_dim=32, - v_head_dim=32, - q_lora_rank=q_lora_rank, - num_hidden_layers=1, - ) - with torch.device("meta"): - return DeepseekV2Attention(cfg, layer_idx=0) - - -def _match(q_lora_rank: int | None) -> MLAFuser | None: - """Match a meta `DeepseekV2Attention` directly (bypassing the per-class - `get_fuser` cache, so both q_lora variants of the same class are seen).""" - attn = _attention(q_lora_rank) - return MLAFuser.match(trace(attn), attn) - - -def test_discovers_modules_without_q_lora(): - fuser = _match(q_lora_rank=None) - assert isinstance(fuser, MLAFuser) - assert fuser.q_proj_name == "q_proj" - assert fuser.kv_a_proj_name == "kv_a_proj_with_mqa" - assert fuser.kv_a_layernorm_name == "kv_a_layernorm" - assert fuser.kv_b_proj_name == "kv_b_proj" - assert fuser.o_proj_name == "o_proj" - assert fuser.q_a_proj_name is None - # Nothing is stacked without a query LoRA. - assert fuser.packed_modules_mapping == {} - assert fuser.orig_to_new_stacked("model.layers.0.self_attn") == {} - - -def test_discovers_modules_with_q_lora(): - fuser = _match(q_lora_rank=64) - assert isinstance(fuser, MLAFuser) - assert fuser.q_a_proj_name == "q_a_proj" - assert fuser.q_a_layernorm_name == "q_a_layernorm" - assert fuser.q_b_proj_name == "q_b_proj" - assert fuser.kv_a_proj_name == "kv_a_proj_with_mqa" - assert fuser.kv_a_layernorm_name == "kv_a_layernorm" - assert fuser.kv_b_proj_name == "kv_b_proj" - assert fuser.o_proj_name == "o_proj" - assert fuser.q_proj_name is None - - -def test_q_lora_stacks_qkv_a_proj(): - """The MLA layer reads `q_a_proj` and `kv_a_proj_with_mqa` fused into one - down-projection, so both checkpoint weights must remap into it.""" - fuser = _match(q_lora_rank=64) - assert isinstance(fuser, MLAFuser) - prefix = "model.layers.0.self_attn" - merged = f"{prefix}.{_FUSED_QKV_A_PROJ}" - assert fuser.packed_modules_mapping == { - _FUSED_QKV_A_PROJ: ["q_a_proj", "kv_a_proj_with_mqa"] - } - assert fuser.orig_to_new_stacked(prefix) == { - f"{prefix}.q_a_proj": (merged, 0), - f"{prefix}.kv_a_proj_with_mqa": (merged, 1), - } - - -@pytest.mark.parametrize("q_lora_rank", [64, None]) -def test_update_forward_rewrites_real_attention(q_lora_rank): - """The source rewrite must survive the input-stability check. - - The q-LoRA path hoists the merged down-projection to a top-level statement - while its calls sit inside the `q_lora_rank` branches, so the region it - spans is wide; a check that over-approximates aliasing would silently refuse - here and cost the fusion rather than fail loudly.""" - fuser = _match(q_lora_rank) - assert isinstance(fuser, MLAFuser) - fuser.update_forward(_attention(q_lora_rank)) - names = set(fuser.fused_forward.__code__.co_names) - if fuser.q_a_proj_name is not None: - assert _FUSED_QKV_A_PROJ in names - assert not {"q_a_proj", "kv_a_proj_with_mqa"} & names - - -class _Norm(nn.Module): - """A real RMSNorm computation: `match` verifies chain norms via `RMSNormFuser`.""" - - def __init__(self, n: int): - super().__init__() - self.weight = nn.Parameter(torch.ones(n)) - - def forward(self, x): - variance = x.pow(2).mean(-1, keepdim=True) - return self.weight * (x * torch.rsqrt(variance + 1e-6)) - - -class RenamedMLA(nn.Module): - """An MLA-shaped attention whose children have non-standard names, proving - discovery is by structure and not attribute name.""" - - def __init__(self): - super().__init__() - heads, kv_lora, rope, nope, v, hidden = 4, 32, 8, 8, 16, 64 - self.alpha = nn.Linear(hidden, heads * (nope + rope)) # q_proj - self.beta = nn.Linear(hidden, kv_lora + rope) # kv_a_proj_with_mqa - self.gamma = _Norm(kv_lora) # kv_a_layernorm - self.delta = nn.Linear(kv_lora, heads * (nope + v)) # kv_b_proj - self.omega = nn.Linear(heads * v, hidden) # o_proj (uncalled) - self.kv_lora, self.rope = kv_lora, rope - - def forward(self, hidden_states): - q = self.alpha(hidden_states) - kv_lora, k_pe = torch.split( - self.beta(hidden_states), [self.kv_lora, self.rope], dim=-1 - ) - expanded = self.delta(self.gamma(kv_lora)) - # Stand-in for the attention interface; `match` finds `o_proj` (omega) from - # the source as the Linear producing the returned value. - attn_output = expanded.sum() + q.sum() + k_pe.sum() - attn_output = self.omega(attn_output) - return attn_output - - -def test_discovers_modules_under_arbitrary_names(): - """Discovery is purely structural: `RenamedMLA` gives its children non-standard - names, and `match` still locates each projection by dataflow.""" - with torch.device("meta"): - module = RenamedMLA() - fuser = MLAFuser.match(trace(module), module) - assert isinstance(fuser, MLAFuser) - assert fuser.q_a_proj_name is None - assert fuser.q_proj_name == "alpha" - assert fuser.kv_a_proj_name == "beta" - assert fuser.kv_a_layernorm_name == "gamma" - assert fuser.kv_b_proj_name == "delta" - assert fuser.o_proj_name == "omega" - - -class GLU(nn.Module): - """A gated MLP: matches the GLU fuser, never MLA.""" - - def __init__(self): - super().__init__() - self.gate = nn.Linear(16, 16) - self.up = nn.Linear(16, 16) - - def forward(self, x): - return self.up(F.silu(self.gate(x))) - - -def test_non_mla_is_not_matched(): - with torch.device("meta"): - assert get_fuser(GLU(), MLAFuser) is None diff --git a/tests/models/transformers/fusers/test_moe.py b/tests/models/transformers/fusers/test_moe.py deleted file mode 100644 index 284b1d6a09..0000000000 --- a/tests/models/transformers/fusers/test_moe.py +++ /dev/null @@ -1,611 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Unit tests for the Transformers modeling backend's MoE fuser.""" - -import pytest -import torch -import torch.nn as nn -import torch.nn.functional as F - -from vllm.model_executor.models.transformers.fusers import MoEBlockFuser - -from .test_linear import GLUMLP - - -class TopKRouter(nn.Module): - """HF v5 top-k router: `linear -> softmax -> topk (-> renorm)`.""" - - def __init__(self, num_experts=8, hidden=16, top_k=2, sigmoid=False): - super().__init__() - self.top_k = top_k - self.sigmoid = sigmoid - self.weight = nn.Parameter(torch.zeros(num_experts, hidden)) - - def forward(self, hidden_states): - logits = F.linear(hidden_states, self.weight) - scores = torch.sigmoid(logits) if self.sigmoid else F.softmax(logits, dim=-1) - value, index = torch.topk(scores, self.top_k, dim=-1) - value = value / value.sum(dim=-1, keepdim=True) - return logits, value, index - - -class SelectionScoredRouter(TopKRouter): - """Scores the selected logits only (GraniteMoE/Aria style), in fp32.""" - - def forward(self, hidden_states): - logits = F.linear(hidden_states, self.weight).float() - value, index = torch.topk(logits, self.top_k, dim=-1) - value = ( - torch.sigmoid(value) if self.sigmoid else F.softmax(value, dim=-1) - ).type_as(hidden_states) - return logits, value, index - - -class EpsilonRenormRouter(TopKRouter): - """Renormalizes via an epsilon-offset denominator (DeepSeek-V3 style).""" - - def forward(self, hidden_states): - logits = F.linear(hidden_states, self.weight) - scores = torch.sigmoid(logits) - value, index = torch.topk(scores, self.top_k, dim=-1) - denominator = value.sum(dim=-1, keepdim=True) + 1e-20 - value /= denominator - return logits, value, index - - -class UntraceableRouter(TopKRouter): - """Scores and selects, then loses the trace before the renormalization.""" - - def forward(self, hidden_states): - logits = F.linear(hidden_states, self.weight) - scores = F.softmax(logits, dim=-1) - value, index = torch.topk(scores, self.top_k, dim=-1) - if value.sum() > 0: # data-dependent branch: tracing stops here - value = value / value.sum(dim=-1, keepdim=True) - return logits, value, index - - -class ScaledRouter(TopKRouter): - """Greedy router scaling its top-k weights (DeepSeek `routed_scaling_factor`).""" - - def forward(self, hidden_states): - logits = F.linear(hidden_states, self.weight) - scores = F.softmax(logits, dim=-1) - value, index = torch.topk(scores, self.top_k, dim=-1) - value = value * 16.0 - return logits, value, index - - -class Fp32Router(TopKRouter): - """Router that computes its logits in fp32 (DeepSeek/GLM style).""" - - def forward(self, hidden_states): - logits = F.linear( - hidden_states.type(torch.float32), self.weight.type(torch.float32) - ) - scores = F.softmax(logits, dim=-1) - value, index = torch.topk(scores, self.top_k, dim=-1) - return logits, value.to(hidden_states.dtype), index - - -class GroupedRouter(TopKRouter): - """Group-limited router (DeepSeek `group_limited_greedy`), scaled weights.""" - - def forward(self, hidden_states): - logits = F.linear(hidden_states, self.weight) - scores = F.softmax(logits, dim=-1) - group_scores = scores.view(-1, 4, 2).max(dim=-1).values - group_idx = torch.topk(group_scores, k=2, dim=-1)[1] - group_mask = torch.zeros_like(group_scores) - group_mask.scatter_(1, group_idx, 1) - score_mask = group_mask.unsqueeze(-1).expand(-1, 4, 2).reshape(-1, 8) - scores = scores.masked_fill(~score_mask.bool(), 0.0) - value, index = torch.topk(scores, self.top_k, dim=-1) - value = value * 16.0 - return logits, value, index - - -class CorrectionRouter(nn.Module): - """Router with a score-correction bias buffer (DeepSeek-V3 noaux) -> matched.""" - - def __init__(self, num_experts=8, hidden=16): - super().__init__() - self.weight = nn.Parameter(torch.zeros(num_experts, hidden)) - self.register_buffer("e_score_correction_bias", torch.zeros(num_experts)) - - def forward(self, hidden_states): - logits = F.linear(hidden_states, self.weight) - scores = torch.sigmoid(logits) + self.e_score_correction_bias - _, index = torch.topk(scores, 2, dim=-1) - return logits, scores, index - - -class BiasedRouter(TopKRouter): - """A valid top-k router but not `weight`-only (extra `bias` param) -> declined.""" - - def __init__(self): - super().__init__() - self.bias = nn.Parameter(torch.zeros(8)) - - def forward(self, hidden_states): - logits = F.linear(hidden_states, self.weight) + self.bias - scores = F.softmax(logits, dim=-1) - value, index = torch.topk(scores, self.top_k, dim=-1) - return logits, value, index - - -class DisconnectedRouter(TopKRouter): - """linear+softmax+top-k present but top-k ignores the logits -> not a router.""" - - def forward(self, hidden_states): - logits = F.linear(hidden_states, self.weight) - _ = F.softmax(logits, dim=-1) # scored, but not consumed by top-k - value, index = torch.topk(hidden_states, self.top_k, dim=-1) - return logits, value, index - - -class ExtraInputRouter(TopKRouter): - """Router taking a block-owned correction bias as a second input (HYV3).""" - - def forward(self, hidden_states, bias=0.0): - logits = F.linear(hidden_states, self.weight) - _, index = torch.topk(logits.sigmoid() + bias, self.top_k, dim=-1) - value = logits.sigmoid().gather(1, index) - return logits, value, index - - -class MoEExperts(nn.Module): - """Packed experts (3D weights); only its name (`experts`) matters here.""" - - def __init__(self, num_experts=8, hidden=16, inter=32): - super().__init__() - self.gate_up_proj = nn.Parameter(torch.zeros(num_experts, 2 * inter, hidden)) - self.down_proj = nn.Parameter(torch.zeros(num_experts, hidden, inter)) - - def forward(self, hidden_states, index, weights): - return hidden_states - - -class MoEBlock(nn.Module): - """Single-tensor MoE block (Qwen3-style); subclasses override `_shared`.""" - - def __init__(self, router_cls=TopKRouter): - super().__init__() - self.experts = MoEExperts() - self.gate = router_cls() - - def _shared(self, x, logits): - """The term added to the experts' output (none for a plain block).""" - return None - - def forward(self, hidden_states): - x = hidden_states.reshape(-1, hidden_states.shape[-1]) - logits, weights, index = self.gate(x) - out = self.experts(x, index, weights) - if (shared := self._shared(x, logits)) is not None: - out = out + shared - return out.reshape(hidden_states.shape) - - -class MoEBlockNoShared(MoEBlock): - """No shared-expert child but a gate-derived add -> declined, not misread.""" - - def _shared(self, x, logits): - return logits.sum() - - -class MoEBlockShared(MoEBlock): - """A block with a shared expert and its sigmoid gate (Qwen2-style).""" - - def __init__(self): - super().__init__() - self.shared_expert = GLUMLP() - self.shared_expert_gate = nn.Linear(16, 1, bias=False) - - def _shared(self, x, logits): - return torch.sigmoid(self.shared_expert_gate(x)) * self.shared_expert(x) - - -class MoEBlockSharedNoGate(MoEBlock): - """A block with an ungated shared expert -> native, shared passed through.""" - - def __init__(self): - super().__init__() - self.shared_expert = GLUMLP() - - def _shared(self, x, logits): - return self.shared_expert(x) - - -class MoEBlockTuple(MoEBlock): - """A tuple-returning block (gpt-oss-style) -> must decline.""" - - def forward(self, hidden_states): - x = hidden_states.reshape(-1, hidden_states.shape[-1]) - _, weights, index = self.gate(x) - return self.experts(x, index, weights), index - - -class MoEBlockTupleVar(MoEBlock): - """Returns a name bound to a tuple, not a literal tuple -> must still decline.""" - - def forward(self, hidden_states): - x = hidden_states.reshape(-1, hidden_states.shape[-1]) - _, weights, index = self.gate(x) - result = self.experts(x, index, weights), index - return result - - -class MoEBlockNestedTupleReturn(MoEBlock): - """Tuple `return` in a nested helper; block returns one tensor -> still fuses.""" - - def forward(self, hidden_states): - def keep(a, b): - return a, b - - x = hidden_states.reshape(-1, hidden_states.shape[-1]) - _, weights, index = self.gate(x) - out, _ = keep(self.experts(x, index, weights), index) - return out.reshape(hidden_states.shape) - - -class PlainMLP(nn.Module): - """A non-GLU FFN: `down(act(up(x)))`, no gating multiply.""" - - def __init__(self, hidden: int = 16, inter: int = 32): - super().__init__() - self.up_proj = nn.Linear(hidden, inter, bias=False) - self.down_proj = nn.Linear(inter, hidden, bias=False) - self.act_fn = nn.SiLU() - - def forward(self, x): - return self.down_proj(self.act_fn(self.up_proj(x))) - - -class MoEBlockSharedNonGLU(MoEBlock): - """A non-GLU shared expert -> detected by dataflow (no gate/up merge).""" - - def __init__(self): - super().__init__() - self.shared_expert = PlainMLP() - - def _shared(self, x, logits): - return self.shared_expert(x) - - -class MoEBlockUnaccounted(MoEBlock): - """A weight-bearing child outside the fused dataflow (pre-router) -> declined.""" - - def __init__(self): - super().__init__() - self.extra = nn.Linear(16, 16, bias=False) - - def forward(self, hidden_states): - x = self.extra(hidden_states.reshape(-1, hidden_states.shape[-1])) - _, weights, index = self.gate(x) - return self.experts(x, index, weights).reshape(hidden_states.shape) - - -class BufferScale(nn.Module): - """A stateful child carrying only a buffer (no parameters).""" - - def __init__(self, hidden: int = 16): - super().__init__() - self.register_buffer("scale", torch.ones(hidden)) - - def forward(self, x): - return x * self.scale - - -class MoEBlockUnaccountedBuffer(MoEBlockUnaccounted): - """Like `MoEBlockUnaccounted`, but the extra child holds only a buffer.""" - - def __init__(self): - super().__init__() - self.extra = BufferScale() - - -class MoEBlockExpertsKeywords(MoEBlock): - """Passes the experts' routing by keyword -> still fuses.""" - - def forward(self, hidden_states): - x = hidden_states.reshape(-1, hidden_states.shape[-1]) - _, weights, index = self.gate(x) - return self.experts(x, weights=weights, index=index) - - -class ScaledExperts(MoEExperts): - def forward(self, hidden_states, index, weights, scale=1.0): - return hidden_states * scale - - -class MoEBlockExpertsExtraArg(MoEBlock): - """Passes the experts an extra argument the fused forward would drop.""" - - def __init__(self): - super().__init__() - self.experts = ScaledExperts() - - def forward(self, hidden_states): - x = hidden_states.reshape(-1, hidden_states.shape[-1]) - _, weights, index = self.gate(x) - return self.experts(x, index, weights, scale=2.0) - - -class MoEBlockSharedUnflattened(MoEBlockSharedNoGate): - """Gate and shared expert read the unflattened input (Kolibri-1).""" - - def forward(self, hidden_states): - _, weights, index = self.gate(hidden_states) - x = hidden_states.view(-1, hidden_states.shape[-1]) - out = self.experts(x, index, weights).view(*hidden_states.shape) - return out + self.shared_expert(hidden_states) - - -class MoEBlockSharedAveraged(MoEBlockSharedUnflattened): - """Averages the experts' and shared expert's outputs (Cohere2-MoE).""" - - def forward(self, hidden_states): - return super().forward(hidden_states) / 2 - - -class MoEBlockScaled(MoEBlockSharedNoGate): - """Scales the experts' output by `routed_scaling_factor` (Laguna).""" - - def forward(self, hidden_states): - x = hidden_states.reshape(-1, hidden_states.shape[-1]) - _, weights, index = self.gate(x) - return self.experts(x, index, weights) * 2.5 + self.shared_expert(x) - - -class MoEBlockScaledBiasedRouter(MoEBlockScaled): - """An unrecognized router with a scaled experts' output.""" - - def __init__(self): - super().__init__() - self.gate = BiasedRouter() - - -class MoEBlockSharedBiasedRouter(MoEBlockSharedUnflattened): - """An unrecognized router with a shared expert.""" - - def __init__(self): - super().__init__() - self.gate = BiasedRouter() - - -class MoEBlockGateExtraInput(MoEBlock): - """Passes the gate a block-owned bias the fused forward would drop (HYV3).""" - - def __init__(self): - super().__init__(ExtraInputRouter) - self.register_buffer("e_score_correction_bias", torch.zeros(8)) - - def forward(self, hidden_states): - x = hidden_states.reshape(-1, hidden_states.shape[-1]) - _, weights, index = self.gate(x, self.e_score_correction_bias) - return self.experts(x, index, weights) - - -class MoEBlockCastCombine(MoEBlockSharedNoGate): - """Combines the outputs in fp32 (HYV3).""" - - def forward(self, hidden_states): - x = hidden_states.reshape(-1, hidden_states.shape[-1]) - _, weights, index = self.gate(x) - out = self.experts(x, index, weights).float() + self.shared_expert(x).float() - return out.to(x.dtype) - - -class MoEBlockIdentityLatent(MoEBlockSharedNoGate): - """Wraps the experts in unused (identity) latent projections (NemotronH).""" - - def __init__(self): - super().__init__() - self.fc1_latent_proj = nn.Identity() - self.fc2_latent_proj = nn.Identity() - - def forward(self, hidden_states): - x = hidden_states.reshape(-1, hidden_states.shape[-1]) - _, weights, index = self.gate(x) - out = self.fc2_latent_proj( - self.experts(self.fc1_latent_proj(x), index, weights) - ) - return out + self.shared_expert(x) - - -@pytest.mark.parametrize("sigmoid", [False, True]) -def test_moe_fuser_detects_router(sigmoid): - with torch.device("meta"): - block = MoEBlock(lambda: TopKRouter(sigmoid=sigmoid)) - fuser = MoEBlockFuser.match(block, "experts") - assert isinstance(fuser, MoEBlockFuser) - assert fuser.gate_name == "gate" - assert fuser.scoring_func == ("sigmoid" if sigmoid else "softmax") - assert fuser.shared_name is None and fuser.shared_gate_name is None - - -@pytest.mark.parametrize("sigmoid", [False, True]) -def test_moe_fuser_matches_router_scoring_the_selection(sigmoid): - """Scoring the selection rather than all logits still allows the fuser to match.""" - with torch.device("meta"): - block = MoEBlock(lambda: SelectionScoredRouter(sigmoid=sigmoid)) - fuser = MoEBlockFuser.match(block, "experts") - assert isinstance(fuser, MoEBlockFuser) - assert fuser.gate_name == "gate" - assert fuser.scoring_func == ("sigmoid" if sigmoid else "softmax") - assert fuser.renormalize is (not sigmoid) - assert fuser.router_dtype == torch.float32 - - -def test_moe_fuser_matches_scaled_router(): - """Weight scaling after the top-k (DeepSeek style) does not break matching.""" - with torch.device("meta"): - block = MoEBlock(ScaledRouter) - assert isinstance(MoEBlockFuser.match(block, "experts"), MoEBlockFuser) - - -@pytest.mark.parametrize( - "router_cls,expected", - [ - (TopKRouter, True), # divides by the sum of the selection - (EpsilonRenormRouter, True), # ... with an epsilon in the denominator - (ScaledRouter, False), # scales the weights instead of renormalizing - (UntraceableRouter, None), # trace stops before it could say - ], -) -def test_moe_fuser_reads_renormalize_from_the_gate(router_cls, expected): - """The router's graph, not the config, decides whether vLLM renormalizes: it - is the code being replaced, and it already bakes in `norm_topk_prob` because - the flag is a Python bool that fx resolves at trace time. Absence of the - division only means "no" on a complete trace.""" - with torch.device("meta"): - block = MoEBlock(router_cls) - fuser = MoEBlockFuser.match(block, "experts") - assert isinstance(fuser, MoEBlockFuser) - assert fuser.renormalize is expected - - -def test_moe_fuser_matches_grouped_router(): - """Group masking between the score and the routing top-k still matches: the - scorer is anchored on the routing (last) top-k, not the group one.""" - with torch.device("meta"): - block = MoEBlock(GroupedRouter) - fuser = MoEBlockFuser.match(block, "experts") - assert isinstance(fuser, MoEBlockFuser) - assert fuser.scoring_func == "softmax" - - -def test_moe_fuser_matches_correction_router(): - """A score-correction bias buffer (DeepSeek-V3 noaux) is allowed; the - rebuilt gate carries it in fp32 for FusedMoE's biased routers.""" - with torch.device("meta"): - block = MoEBlock(CorrectionRouter) - fuser = MoEBlockFuser.match(block, "experts") - assert isinstance(fuser, MoEBlockFuser) - assert fuser.scoring_func == "sigmoid" - - -def test_moe_fuser_reads_router_dtype_from_the_gate(): - """A router that computes its logits in fp32 must keep routing in fp32 when - rebuilt, even though no config field names the dtype. The cast back to the - activation dtype after the top-k must not be mistaken for the routing dtype.""" - with torch.device("meta"): - fp32 = MoEBlockFuser.match(MoEBlock(Fp32Router), "experts") - default = MoEBlockFuser.match(MoEBlock(TopKRouter), "experts") - assert fp32.router_dtype == torch.float32 - assert default.router_dtype is None - - -def test_moe_fuser_detects_shared_experts(): - with torch.device("meta"): - block = MoEBlockShared() - fuser = MoEBlockFuser.match(block, "experts") - assert isinstance(fuser, MoEBlockFuser) - assert fuser.shared_name == "shared_expert" - assert fuser.shared_gate_name == "shared_expert_gate" - - -def test_moe_fuser_shared_without_gate(): - with torch.device("meta"): - block = MoEBlockSharedNoGate() - fuser = MoEBlockFuser.match(block, "experts") - assert isinstance(fuser, MoEBlockFuser) - assert fuser.shared_name == "shared_expert" - assert fuser.shared_gate_name is None - - -def test_moe_fuser_detects_non_glu_shared_expert(): - with torch.device("meta"): - block = MoEBlockSharedNonGLU() - fuser = MoEBlockFuser.match(block, "experts") - assert isinstance(fuser, MoEBlockFuser) - # Recognised by dataflow (added to the experts' output), though not a GLU. - assert fuser.shared_name == "shared_expert" - assert fuser.shared_gate_name is None - - -@pytest.mark.parametrize( - "block_cls", - [ - lambda: MoEBlock(BiasedRouter), # router with an unrecognized extra param - MoEBlockTuple, # tuple-returning block (e.g. gpt-oss) - MoEBlockTupleVar, # tuple returned via a name binding, not a literal - MoEBlockUnaccounted, # weight-bearing child outside the fused dataflow - MoEBlockUnaccountedBuffer, # buffer-only child outside the fused dataflow - MoEBlockExpertsExtraArg, # extra experts argument the fused forward drops - MoEBlockNoShared, # gate-derived term added to the experts' output - MoEBlockSharedAveraged, # experts' and shared expert's outputs averaged - MoEBlockGateExtraInput, # gate input the fused forward drops - ], -) -def test_moe_fuser_declines_unsupported(block_cls): - with torch.device("meta"): - block = block_cls() - assert MoEBlockFuser.match(block, "experts") is None - - -def test_moe_fuser_ignores_nested_returns(): - """A tuple `return` inside a nested helper must not decline a block whose own - forward returns a single tensor.""" - with torch.device("meta"): - block = MoEBlockNestedTupleReturn() - assert isinstance(MoEBlockFuser.match(block, "experts"), MoEBlockFuser) - - -def test_moe_fuser_router_requires_connected_dataflow(): - """A gate with linear + softmax + top-k present but not wired as a router - (top-k selects over the input, not the scored logits) is not detected.""" - with torch.device("meta"): - block = MoEBlock(DisconnectedRouter) - assert MoEBlockFuser.match(block, "experts") is None - - -def test_moe_fuser_unrecognized_router_moves_shared_expert(): - """Without vLLM routing, the shared expert still moves to vLLM.""" - with torch.device("meta"): - block = MoEBlockSharedBiasedRouter() - fuser = MoEBlockFuser.match(block, "experts") - assert fuser.scoring_func is None - assert fuser.shared_name == "shared_expert" - - -@pytest.mark.parametrize( - ("block_cls", "scoring_func"), - [ - (MoEBlockSharedUnflattened, None), # routing falls back to Transformers - (MoEBlock, "softmax"), # no shared output to divide - ], -) -def test_moe_fuser_divided_shared_output(block_cls, scoring_func): - with torch.device("meta"): - fuser = MoEBlockFuser.match(block_cls(), "experts", divides_shared_output=True) - assert fuser.scoring_func == scoring_func - - -def test_moe_fuser_scaled_output_needs_vllm_routing(): - """Only vLLM routing applies the routed scale, so nothing else moves without it.""" - with torch.device("meta"): - block = MoEBlockScaledBiasedRouter() - assert MoEBlockFuser.match(block, "experts") is None - - -def test_moe_fuser_matches_experts_called_by_keyword(): - with torch.device("meta"): - block = MoEBlockExpertsKeywords() - assert isinstance(MoEBlockFuser.match(block, "experts"), MoEBlockFuser) - - -def test_moe_fuser_scaled_output_with_vllm_routing(): - """A scaled block fuses because vLLM routing applies `routed_scaling_factor`.""" - with torch.device("meta"): - fuser = MoEBlockFuser.match(MoEBlockScaled(), "experts") - assert fuser.scoring_func == "softmax" - assert fuser.shared_name == "shared_expert" - - -@pytest.mark.parametrize("block_cls", [MoEBlockCastCombine, MoEBlockIdentityLatent]) -def test_moe_fuser_looks_through_casts_and_identity(block_cls): - with torch.device("meta"): - fuser = MoEBlockFuser.match(block_cls(), "experts") - assert fuser.shared_name == "shared_expert" diff --git a/tests/models/transformers/fusers/test_rms_norm.py b/tests/models/transformers/fusers/test_rms_norm.py deleted file mode 100644 index 256f83649a..0000000000 --- a/tests/models/transformers/fusers/test_rms_norm.py +++ /dev/null @@ -1,409 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Unit tests for the Transformers modeling backend's RMSNorm fuser.""" - -from types import SimpleNamespace - -import pytest -import torch -import torch.nn as nn -import torch.nn.functional as F - -from vllm.model_executor.models.transformers.fuser import get_fuser -from vllm.model_executor.models.transformers.fusers import RMSNormFuser - - -class RMSNorm(nn.Module): - """The canonical HF RMSNorm: `weight * x * rsqrt(mean(x**2) + eps)`.""" - - def __init__(self, hidden: int = 16, eps: float = 1e-5, weight: bool = True): - super().__init__() - if weight: - self.weight = nn.Parameter(torch.ones(hidden)) - self.variance_epsilon = eps - - def _rms(self, x): - return x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.variance_epsilon) - - def forward(self, x): - return self.weight * self._rms(x.to(torch.float32)).to(x.dtype) - - -class GemmaRMSNorm(RMSNorm): - """Zero-centered weight: `(1 + weight) * normalized`.""" - - def __init__(self, hidden: int = 16, eps: float = 1e-6): - super().__init__(hidden, eps) - self.weight = nn.Parameter(torch.zeros(hidden)) - - def forward(self, x): - return (1.0 + self.weight) * self._rms(x.to(torch.float32)).to(x.dtype) - - -class WeightlessRMSNorm(RMSNorm): - """No scale parameter (e.g. Gemma3n `with_scale=False`).""" - - def __init__(self, hidden: int = 16, eps: float = 1e-6): - super().__init__(hidden, eps, weight=False) - - def forward(self, x): - return self._rms(x.to(torch.float32)).to(x.dtype) - - -class Gemma4RMSNorm(RMSNorm): - """`pow(v, -0.5)` where the others write `rsqrt(v)` (HF `Gemma4RMSNorm._norm`). - - Gemma 4 spells the reciprocal square root as `torch.pow(..., -0.5)` to keep - Torch and JAX in agreement, and -- unlike Gemma 1-3 -- its weight is *not* - zero-centered, so this fuses to a plain `RMSNorm`. - """ - - def _rms(self, x): - mean_squared = x.pow(2).mean(-1, keepdim=True) + self.variance_epsilon - return x * torch.pow(mean_squared, -0.5) - - def forward(self, x): - return (self._rms(x.float()) * self.weight.float()).type_as(x) - - -class PowOperatorRMSNorm(Gemma4RMSNorm): - """The `v ** -0.5` operator spelling of the same reciprocal square root.""" - - def _rms(self, x): - return x * (x.pow(2).mean(-1, keepdim=True) + self.variance_epsilon) ** -0.5 - - -class KwargPowRMSNorm(Gemma4RMSNorm): - """The keyword spelling `torch.pow(v, exponent=-0.5)` of the same rsqrt.""" - - def _rms(self, x): - mean_squared = x.pow(2).mean(-1, keepdim=True) + self.variance_epsilon - return x * torch.pow(mean_squared, exponent=-0.5) - - -class KwargBasePowRMSNorm(Gemma4RMSNorm): - """The fully functional spelling `torch.pow(input=v, exponent=-0.5)`.""" - - def _rms(self, x): - mean_squared = x.pow(2).mean(-1, keepdim=True) + self.variance_epsilon - return x * torch.pow(input=mean_squared, exponent=-0.5) - - -class WeightlessGemma4RMSNorm(Gemma4RMSNorm): - """Gemma 4 with `with_scale=False`: the `pow` spelling and no scale parameter.""" - - def __init__(self, hidden: int = 16, eps: float = 1e-6): - super().__init__(hidden, eps, weight=False) - - def forward(self, x): - return self._rms(x.float()).type_as(x) - - -class LayerNorm(RMSNorm): - """An RMSNorm not named `*RMSNorm`, keeping the input dtype (no upcast).""" - - def __init__(self, hidden: int = 16, eps: float = 1e-6): - super().__init__(hidden, eps) - - def forward(self, x): - return self.weight * self._rms(x) - - -class NamedEpsRMSNorm(nn.Module): - """Holds eps under `attr`, with no `variance_epsilon` to find instead.""" - - attr = "eps" - - def __init__(self, hidden: int = 16, eps: float = 1e-6): - super().__init__() - self.weight = nn.Parameter(torch.ones(hidden)) - setattr(self, self.attr, eps) - - def forward(self, x): - eps = getattr(self, self.attr) - return self.weight * (x * torch.rsqrt(x.pow(2).mean(-1, True) + eps)) - - -class ToleranceRMSNorm(NamedEpsRMSNorm): - """Names eps something other than `eps`, so only its value identifies it.""" - - attr = "tolerance" - - -class LiteralEpsRMSNorm(RMSNorm): - """eps is a literal in the source, and an unrelated attribute happens to - hold the same value: matching on the value alone would bind to it.""" - - def __init__(self, hidden: int = 16, eps: float = 1e-5): - super().__init__(hidden, eps) - self.epsilon_lookalike = 1e-4 - - def forward(self, x): - return self.weight * (x * torch.rsqrt(x.pow(2).mean(-1, True) + 1e-4)) - - -class AmbiguousEpsRMSNorm(nn.Module): - """Two attributes hold the eps value, and the forward reads the second, so - their order cannot pick it.""" - - def __init__(self, hidden: int = 16, eps: float = 1e-6): - super().__init__() - self.weight = nn.Parameter(torch.ones(hidden)) - self.decoy_eps = eps - self.variance_epsilon = eps - - def forward(self, x): - var = x.pow(2).mean(-1, True) - return self.weight * (x * torch.rsqrt(var + self.variance_epsilon)) - - -class NotAnRMSNorm(RMSNorm): - """Mean-subtracting LayerNorm-like math -> not an RMSNorm.""" - - def __init__(self, hidden: int = 16, eps: float = 1e-6): - super().__init__(hidden, eps) - - def forward(self, x): - x = x - x.mean(-1, keepdim=True) - variance = x.var(-1, keepdim=True) - return self.weight * x / torch.sqrt(variance + self.variance_epsilon) - - -class GatedRMSNorm(RMSNorm): - """Second input and tail compute -> not an RMSNorm.""" - - def forward(self, x, gate=None): - normed = self.weight * self._rms(x.to(torch.float32)).to(x.dtype) - return normed * F.silu(gate) - - -class GatedFusedRMSNorm(nn.Module): - """Same as GatedRMSNorm, but built on the fused `rms_norm` op -> not an RMSNorm.""" - - def __init__(self, hidden: int = 16, eps: float = 1e-5): - super().__init__() - self.weight = nn.Parameter(torch.ones(hidden)) - self.eps = eps - - def forward(self, x, gate=None): - return F.rms_norm(x, (x.shape[-1],), self.weight, self.eps) * F.silu(gate) - - -class UntraceableGatedRMSNorm(RMSNorm): - """Tracer can't see tail compute in forward, but still has a second input (gate).""" - - def forward(self, x, gate=None): - normed = self.weight * self._rms(x.to(torch.float32)).to(x.dtype) - if gate.sum() > 0: # untraceable -> partial graph, no visible tail - normed = normed * F.silu(gate) - return normed - - -@pytest.mark.parametrize( - "cls,eps,zero_centered", - [ - (RMSNorm, 1e-5, False), - (GemmaRMSNorm, 1e-6, True), - (WeightlessRMSNorm, 1e-6, False), - (Gemma4RMSNorm, 1e-6, False), - (PowOperatorRMSNorm, 1e-6, False), - (KwargPowRMSNorm, 1e-6, False), - (KwargBasePowRMSNorm, 1e-6, False), - (WeightlessGemma4RMSNorm, 1e-6, False), - (LayerNorm, 1e-6, False), - (torch.nn.RMSNorm, 1e-5, False), # fused `F.rms_norm` op - ], -) -def test_detects_rms_norm_variants(cls, eps, zero_centered): - with torch.device("meta"): - fuser = get_fuser(cls(16, eps=eps), RMSNormFuser) - assert isinstance(fuser, RMSNormFuser) - assert fuser.zero_centered == zero_centered - - -@pytest.mark.parametrize("cls", [NotAnRMSNorm, nn.LayerNorm, nn.SiLU]) -def test_non_rms_norms_are_not_matched(cls): - with torch.device("meta"): - module = cls(16) if cls is nn.LayerNorm else cls() - assert get_fuser(module, RMSNormFuser) is None - - -@pytest.mark.parametrize( - "cls", [GatedRMSNorm, GatedFusedRMSNorm, UntraceableGatedRMSNorm] -) -def test_gated_rms_norm_is_not_fused(cls): - with torch.device("meta"): - assert get_fuser(cls(), RMSNormFuser) is None - - -@pytest.mark.parametrize( - "cls,expected,zero_centered", - [ - (RMSNorm, "RMSNorm", False), - (GemmaRMSNorm, "GemmaRMSNorm", True), - (WeightlessRMSNorm, "RMSNorm", False), - (Gemma4RMSNorm, "RMSNorm", False), - (KwargPowRMSNorm, "RMSNorm", False), - (KwargBasePowRMSNorm, "RMSNorm", False), - (WeightlessGemma4RMSNorm, "RMSNorm", False), - ], -) -def test_rms_norm_builds_vllm_class(cls, expected, zero_centered, default_vllm_config): - from vllm.model_executor.layers.layernorm import GemmaRMSNorm as VLLMGemmaRMSNorm - from vllm.model_executor.layers.layernorm import RMSNorm as VLLMRMSNorm - - with torch.device("meta"): - module = cls() - fuser = get_fuser(module, RMSNormFuser) - built = fuser.fuse(module, "norm", default_vllm_config) - from vllm.model_executor.models.transformers.fusers.rms_norm import ( - TPAwareNormMixin, - ) - - types_by_name = {"RMSNorm": VLLMRMSNorm, "GemmaRMSNorm": VLLMGemmaRMSNorm} - assert isinstance(built, types_by_name[expected]) - assert isinstance(built, TPAwareNormMixin) - assert built.variance_epsilon == module.variance_epsilon - # The weight states the hidden size (and is what the TP check reads); a norm - # without one has none to state. - weight = getattr(module, "weight", None) - assert isinstance(built.weight, nn.Parameter) == (weight is not None) - assert built.weight.shape[0] == (weight.size(0) if weight is not None else 0) - - -@pytest.mark.parametrize( - "cls", [Gemma4RMSNorm, PowOperatorRMSNorm, KwargPowRMSNorm, KwargBasePowRMSNorm] -) -def test_pow_spelled_norm_fuses_to_equivalent_math(cls, default_vllm_config): - """Matching `pow(v, -0.5)` is only sound if it really is the reciprocal square - root: the fused norm must reproduce the unfused forward, not just replace it. - `pow` and `rsqrt` take different paths to the same value, so compare at float32 - tolerance rather than bit-for-bit.""" - torch.manual_seed(0) - module = cls(16) - with torch.no_grad(): - module.weight.copy_(torch.randn(16)) - built = get_fuser(module, RMSNormFuser).fuse(module, "norm", default_vllm_config) - with torch.no_grad(): - built.weight.copy_(module.weight) - x = torch.randn(4, 16) - torch.testing.assert_close(built(x), module(x), rtol=1e-5, atol=1e-6) - - -def test_weightless_norm_has_no_hidden_size(default_vllm_config): - """A weightless norm states no hidden size, and the model's (LM) hidden size - would be wrong for one on a sub-dimension: Llama 4's `qk_norm` normalizes - head_dim, so sizing it at hidden_size made the TP gather reject its input. - It fuses with none, and so normalizes the width it is given at any TP size, - matching the unfused norm and vLLM's native `RMSNorm(hidden_size=head_dim)`. - """ - module = WeightlessRMSNorm(72) - built = get_fuser(module, RMSNormFuser).fuse(module, "norm", default_vllm_config) - assert built.hidden_size == 0 - - built.tp_size = 2 # emulate TP=2 without a real process group - x = torch.randn(4, 72) - torch.testing.assert_close(built(x), module(x)) - - -def test_fused_rms_norm_op_default_eps(default_vllm_config): - """`torch.nn.RMSNorm` (a single `F.rms_norm` call) matches via the fast path; - its default `eps=None` resolves to `finfo(dtype).eps` in `fuse`.""" - from vllm.model_executor.layers.layernorm import RMSNorm as VLLMRMSNorm - - with torch.device("meta"): - module = torch.nn.RMSNorm(16) # forward is a single `F.rms_norm` call - fuser = get_fuser(module, RMSNormFuser) - assert isinstance(fuser, RMSNormFuser) - assert not fuser.zero_centered - vllm_config = SimpleNamespace(model_config=SimpleNamespace(dtype=torch.float32)) - built = fuser.fuse(module, "norm", vllm_config) - assert isinstance(built, VLLMRMSNorm) - assert built.variance_epsilon == torch.finfo(torch.float32).eps - - -def test_eps_is_derived_per_instance(default_vllm_config): - """Two instances of the same norm class with different eps must fuse to their - own eps: the type-cached fuser holds only structure, not this value.""" - with torch.device("meta"): - for eps in (1e-5, 1e-6): - module = RMSNorm(16, eps=eps) - fuser = get_fuser(module, RMSNormFuser) - built = fuser.fuse(module, "norm", default_vllm_config) - assert built.variance_epsilon == eps - - -@pytest.mark.parametrize("cls", [NamedEpsRMSNorm, ToleranceRMSNorm]) -def test_eps_attr_is_found_by_value_not_name(cls, default_vllm_config): - """The eps attribute is identified by holding the traced value, so a norm - stays per-instance correct whatever it names it.""" - with torch.device("meta"): - for eps in (1e-5, 1e-6): - module = cls(16, eps=eps) - fuser = get_fuser(module, RMSNormFuser) - assert fuser.eps_attr == cls.attr - built = fuser.fuse(module, "norm", default_vllm_config) - assert built.variance_epsilon == eps - - -def test_literal_eps_is_not_mistaken_for_an_attribute(default_vllm_config, caplog): - """A literal eps is recognised as coming from no attribute, even when one - holds the same value, and is taken from the traced source instead.""" - logger = "vllm.model_executor.models.transformers.fusers.rms_norm" - with caplog.at_level("DEBUG", logger=logger), torch.device("meta"): - module = LiteralEpsRMSNorm() - fuser = get_fuser(module, RMSNormFuser) - built = fuser.fuse(module, "norm", default_vllm_config) - assert fuser.eps_attr is None - assert built.variance_epsilon == 1e-4 - assert "does not hold its eps" in caplog.text - - -def test_ambiguous_eps_attrs_are_disambiguated(default_vllm_config): - """When several attributes hold the eps value, the one the forward actually - reads is identified, and they are all left as they were found.""" - with torch.device("meta"): - module = AmbiguousEpsRMSNorm(16, eps=1e-6) - before = dict(vars(module)) - fuser = get_fuser(module, RMSNormFuser) - assert fuser.eps_attr == "variance_epsilon" - assert vars(module) == before - - other = AmbiguousEpsRMSNorm(16, eps=1e-5) - other.decoy_eps = 1e-3 - built = fuser.fuse(other, "norm", default_vllm_config) - assert built.variance_epsilon == 1e-5 - - -def test_fused_norm_is_gather_capable(default_vllm_config): - """Every weighted fused norm is emitted gather-capable, so a norm on a head-sharded - projection (OLMoE-style) self-corrects at runtime with no QKV-specific - plumbing. A full-width input skips the gather and equals a plain norm.""" - from vllm.model_executor.layers.layernorm import GemmaRMSNorm, RMSNorm - from vllm.model_executor.models.transformers.fusers import rms_norm - - torch.manual_seed(0) - x = torch.randn(4, 16) - for gathered_cls, plain_cls in [ - (rms_norm.TPAwareRMSNorm, RMSNorm), - (rms_norm.TPAwareGemmaRMSNorm, GemmaRMSNorm), - ]: - gathered = gathered_cls(hidden_size=16, eps=1e-6) - assert isinstance(gathered, rms_norm.TPAwareNormMixin) - plain = plain_cls(hidden_size=16, eps=1e-6) - with torch.no_grad(): - weight = torch.randn(16) - gathered.weight.copy_(weight) - plain.weight.copy_(weight) - torch.testing.assert_close(gathered(x), plain(x)) - - -def test_gathered_norm_rejects_uneven_sharding(default_vllm_config): - """A sharded input (narrower than the full-width weight) that does not tile - the weight evenly across ranks is rejected before any collective.""" - from vllm.model_executor.models.transformers.fusers import rms_norm - - norm = rms_norm.TPAwareRMSNorm(hidden_size=8, eps=1e-6) - norm.tp_size = 2 # emulate TP=2 without a real process group - with pytest.raises(ValueError, match="does not tile it evenly"): - norm(torch.randn(2, 3)) # 3 * 2 != 8 diff --git a/tests/models/transformers/test_dynamic_rope.py b/tests/models/transformers/test_dynamic_rope.py deleted file mode 100644 index e47d652846..0000000000 --- a/tests/models/transformers/test_dynamic_rope.py +++ /dev/null @@ -1,86 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Tests for skipping dynamic rope's device sync where it can never rescale.""" - -from types import SimpleNamespace - -import pytest -import torch -from transformers import HunYuanDenseV1Config, LlamaConfig -from transformers.models.hunyuan_v1_dense.modeling_hunyuan_v1_dense import ( - HunYuanDenseV1RotaryEmbedding, -) -from transformers.models.llama.modeling_llama import LlamaRotaryEmbedding - -from vllm.model_executor.models.transformers.utils import ( - can_enable_torch_compile, - disable_noop_dynamic_rope, -) - -MAX_POSITION_EMBEDDINGS = 128 -CONFIG_KWARGS = dict( - hidden_size=64, - num_attention_heads=4, - num_hidden_layers=1, - max_position_embeddings=MAX_POSITION_EMBEDDINGS, -) -DYNAMIC = {"rope_type": "dynamic", "factor": 4.0, "rope_theta": 1e4} - - -def make_vllm_config(hf_config, max_model_len: int) -> SimpleNamespace: - model_config = SimpleNamespace(hf_config=hf_config, max_model_len=max_model_len) - return SimpleNamespace(model_config=model_config) - - -@pytest.mark.parametrize( - "rope_parameters,max_model_len,compilable", - [ - (DYNAMIC, MAX_POSITION_EMBEDDINGS, True), - (DYNAMIC, MAX_POSITION_EMBEDDINGS + 1, False), - # The longrope layer type still syncs - ( - {"full_attention": DYNAMIC, "sliding_attention": {"rope_type": "longrope"}}, - MAX_POSITION_EMBEDDINGS, - False, - ), - ], -) -def test_can_enable_torch_compile(rope_parameters, max_model_len, compilable): - """Dynamic rope only blocks compilation when it can still rescale.""" - config = SimpleNamespace( - max_position_embeddings=MAX_POSITION_EMBEDDINGS, - rope_parameters=rope_parameters, - ) - config.get_text_config = lambda: config - vllm_config = make_vllm_config(config, max_model_len) - assert can_enable_torch_compile(vllm_config) is compilable - - -@pytest.mark.parametrize( - "config,rotary_cls", - [ - (LlamaConfig(**CONFIG_KWARGS, rope_parameters=DYNAMIC), LlamaRotaryEmbedding), - # Hunyuan's dynamic init also applies `alpha`, which must be kept - ( - HunYuanDenseV1Config( - **CONFIG_KWARGS, head_dim=16, rope_parameters=DYNAMIC | {"alpha": 1e3} - ), - HunYuanDenseV1RotaryEmbedding, - ), - ], -) -def test_disable_noop_dynamic_rope(monkeypatch, config, rotary_cls): - """The update no longer syncs and the frequencies are unchanged.""" - inputs = torch.zeros(1, 8, 64), torch.arange(8)[None, :] - rotary_emb = rotary_cls(config) - expected = rotary_emb(*inputs) - - vllm_config = make_vllm_config(config, MAX_POSITION_EMBEDDINGS) - disable_noop_dynamic_rope(rotary_emb, vllm_config) - - def no_sync(*args, **kwargs): - raise AssertionError("dynamic rope update still runs") - - monkeypatch.setattr(torch, "max", no_sync) - actual = rotary_emb(*inputs) - assert all(torch.equal(e, a) for e, a in zip(expected, actual)) diff --git a/tests/models/transformers/test_layer_registry.py b/tests/models/transformers/test_layer_registry.py deleted file mode 100644 index 555d87e379..0000000000 --- a/tests/models/transformers/test_layer_registry.py +++ /dev/null @@ -1,80 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Unit tests for the Transformers backend's hw-agnostic layer resolution. - -`layers._resolve` imports a layer symbol from -`vllm.model_executor.hw_agnostic.layers.` when `VLLM_USE_HW_AGNOSTIC` -is set and the symbol exists, and otherwise falls back to -`vllm.model_executor.layers.`. These tests pin that contract and the -logging that reports which source was used. - -CPU-only: no engine is built. The end-to-end counterpart that serves a model -through `vllm_runner` lives in `test_layer_registry_e2e.py` (GPU job). -""" - -import logging -import sys -import types - -import pytest - -from vllm.model_executor.models.transformers import layers - -HW_MODULE = "vllm.model_executor.hw_agnostic.layers.layernorm" - - -@pytest.fixture -def fake_hw_layernorm(monkeypatch): - """Inject a hw-agnostic `layernorm` module exposing a sentinel `RMSNorm`. - - A `SimpleNamespace` stands in for the module: `importlib.import_module` - returns it from `sys.modules` and `getattr` resolves `RMSNorm`, while its - attributes are set at construction (no `ModuleType` attribute-set that mypy - rejects, no constant `setattr` that ruff rejects).""" - module = types.SimpleNamespace(RMSNorm=type("HwRMSNorm", (), {})) - monkeypatch.setitem(sys.modules, HW_MODULE, module) - return module - - -def test_falls_back_to_vllm_when_disabled(monkeypatch, fake_hw_layernorm): - """Disabled: the vLLM class is used even if a hw-agnostic one exists.""" - monkeypatch.setenv("VLLM_USE_HW_AGNOSTIC", "0") - from vllm.model_executor.layers.layernorm import RMSNorm as VllmRMSNorm - - assert layers._resolve("layernorm", "RMSNorm") is VllmRMSNorm - - -def test_uses_hw_agnostic_when_enabled(monkeypatch, fake_hw_layernorm, caplog_vllm): - """Enabled and available: the hw-agnostic class is used and logged.""" - monkeypatch.setenv("VLLM_USE_HW_AGNOSTIC", "1") - with caplog_vllm.at_level(logging.INFO): - resolved = layers._resolve("layernorm", "RMSNorm") - assert resolved is fake_hw_layernorm.RMSNorm - assert "Using hw-agnostic layer: RMSNorm" in caplog_vllm.text - - -def test_falls_back_when_symbol_missing(monkeypatch, caplog_vllm): - """Enabled but the symbol is not ported: fall back to vLLM and warn.""" - monkeypatch.setenv("VLLM_USE_HW_AGNOSTIC", "1") - # A hw-agnostic module without the requested attribute triggers fallback. - empty = types.ModuleType(HW_MODULE) - monkeypatch.setitem(sys.modules, HW_MODULE, empty) - from vllm.model_executor.layers.layernorm import RMSNorm as VllmRMSNorm - - with caplog_vllm.at_level(logging.WARNING): - resolved = layers._resolve("layernorm", "RMSNorm") - assert resolved is VllmRMSNorm - assert "falling back to default" in caplog_vllm.text - - -def test_act_and_mul_falls_back_for_unknown_activation( - monkeypatch, default_vllm_config -): - """An activation with no hw-agnostic equivalent falls back to vLLM's. - - `default_vllm_config` supplies the config context the CustomOp needs. - """ - monkeypatch.setenv("VLLM_USE_HW_AGNOSTIC", "1") - from vllm.model_executor.layers.activation import GeluAndMul - - assert isinstance(layers.get_act_and_mul_fn("gelu"), GeluAndMul) diff --git a/tests/multimodal/media/test_image.py b/tests/multimodal/media/test_image.py index 1980adf197..574edc8567 100644 --- a/tests/multimodal/media/test_image.py +++ b/tests/multimodal/media/test_image.py @@ -418,20 +418,6 @@ def test_allowed_image_formats_exclude_external_renderer_plugins(): assert denied.isdisjoint(ALLOWED_IMAGE_FORMATS) -def test_kimi_fused_vision_rejects_postscript_payload(): - """Kimi fused vision must use the same image format allowlist.""" - from vllm.transformers_utils.processors.kimi_k25_vision_fused import _to_pil - - eps = b"""%!PS-Adobe-3.0 EPSF-3.0 -%%BoundingBox: 0 0 10 10 -%%EndComments -0 0 moveto 10 10 lineto stroke -showpage -""" - with pytest.raises(Image.UnidentifiedImageError): - _to_pil(eps) - - def test_open_image_does_not_accept_format_override(): """The allowlist helper must not expose a formats override.""" assert "formats" not in inspect.signature(open_image).parameters diff --git a/tests/parser/cohere/test_citations.py b/tests/parser/cohere/test_citations.py deleted file mode 100644 index 14ee1612ea..0000000000 --- a/tests/parser/cohere/test_citations.py +++ /dev/null @@ -1,172 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Citation source resolution and the streamed citation wire shape.""" - -from __future__ import annotations - -from types import SimpleNamespace -from typing import Any - -from vllm.parser.cohere_command import ( - _melody_citations_to_vllm, - _melody_sources_to_vllm, -) - -from .utils import make_parser, stream_parser - - -class TestMelodySourceResolution: - """Pin the parser-side source resolution (see - ``_melody_sources_to_vllm``). The parser receives melody's numeric - ``(tool_call_index, tool_result_indices)`` addressing and resolves - it against a ``position_to_source`` map handed in by - ``CohereServingChatV2._apply_cohere_template_kwargs`` -- the same - map that would otherwise live in the serving layer's resolver. - """ - - @staticmethod - def _fake_melody_source(bucket: int, indices: list[int]) -> Any: - return SimpleNamespace(tool_call_index=bucket, tool_result_indices=indices) - - def test_multi_index_source_fans_out(self): - from vllm.entrypoints.cohere.cohere_chat_message import CitationSource - - position_map: dict[tuple[int, int], CitationSource] = { - (0, 0): CitationSource(type="document", id="d0", document={"id": "d0"}), - (0, 1): CitationSource(type="document", id="d1", document={"id": "d1"}), - } - raw = [self._fake_melody_source(0, [0, 1])] - out = _melody_sources_to_vllm(raw, position_map) - assert [s.id for s in out] == ["d0", "d1"] - # Verify type / payload were plumbed through, not just the id. - assert out[0].type == "document" - assert out[0].document == {"id": "d0"} - - def test_unresolvable_position_skipped(self): - from vllm.entrypoints.cohere.cohere_chat_message import CitationSource - - position_map: dict[tuple[int, int], CitationSource] = { - (0, 0): CitationSource(type="document", id="d0"), - } - raw = [self._fake_melody_source(9, [0])] - assert _melody_sources_to_vllm(raw, position_map) == [] - - def test_missing_position_map_drops_all_sources(self): - # A parser instance without a position map (parser wired - # outside of ``CohereServingChatV2``) can't attribute anything, - # so every source is dropped. Callers downstream will see the - # citation with empty ``sources`` and drop it entirely. - raw = [self._fake_melody_source(0, [0])] - assert _melody_sources_to_vllm(raw, None) == [] - - def test_citations_pass_through_is_thinking_tag(self): - from vllm.entrypoints.cohere.cohere_chat_message import CitationSource - - position_map: dict[tuple[int, int], CitationSource] = { - (0, 0): CitationSource(type="document", id="d0"), - } - raw = [ - SimpleNamespace( - start_index=0, - end_index=5, - text="hello", - is_thinking=True, - sources=[self._fake_melody_source(0, [0])], - ) - ] - out = _melody_citations_to_vllm(raw, position_map) - assert out is not None - assert out[0].type == "THINKING_CONTENT" - assert out[0].sources[0].id == "d0" - - -class TestParserStreamingEndToEnd: - """End-to-end streaming shape: raw model output text is tokenized - with the mock byte-level tokenizer and fed one token at a time - through :meth:`CohereCommand3ReasoningParser.extract_reasoning_streaming`. - We collect the ordered sequence of ``DeltaMessage``s the parser - hands back -- i.e. what the client would receive on the wire -- - and pin the exact shape. - - Combines the three flavors that reach ``delta``: a thinking block - (populates ``reasoning``), a text block (populates ``content``), - and a resolved citation (populates ``citations``) referencing - ``(start, end)`` offsets into the already-streamed content. - """ - - _MODEL_OUTPUT = ( - "<|START_THINKING|>Let me check.<|END_THINKING|>" - "<|START_RESPONSE|>The capital is Paris.<|END_RESPONSE|>" - ) - - def test_full_wire_stream_shape(self, tokenizer, request_obj): - from vllm.entrypoints.cohere.cohere_chat_message import CitationSource - from vllm.renderers.cohere import POSITION_TO_SOURCE_KEY - - # Mimic what ``CohereServingChatV2._apply_cohere_template_kwargs`` - # installs on a request whose tool_call_index=0 tool result at - # index 1 is the "France" document. - position_to_source = { - (0, 1): CitationSource( - type="document", - id="doc-paris", - document={"id": "doc-paris", "title": "France"}, - ), - } - - parser = make_parser( - tokenizer, - "cohere_command3", - chat_template_kwargs={POSITION_TO_SOURCE_KEY: position_to_source}, - ) - deltas = stream_parser(parser, request_obj, tokenizer, self._MODEL_OUTPUT) - - # -- 1. Reasoning and content stream separately and reach - # the wire only as their real payload. Framing tokens - # (``<|START_THINKING|>`` etc.) are consumed by the parser's - # state machine and never produce a delta. - reasoning_stream = [d.reasoning for d in deltas if d.reasoning is not None] - content_stream = [d.content for d in deltas if d.content is not None] - citation_deltas = [d for d in deltas if getattr(d, "citations", None)] - - assert "".join(reasoning_stream) == "Let me check." - assert "".join(content_stream) == "The capital is Paris." - - # -- 2. Exactly one citation reaches the wire, resolved to the - # source from ``position_to_source`` (not the raw - # ``(bucket, idx)`` coordinates). - assert len(citation_deltas) == 1 - (cite_delta,) = citation_deltas - assert cite_delta.citations is not None - assert len(cite_delta.citations) == 1 - cite = cite_delta.citations[0] - assert cite.text == "Paris" - # Char offsets are into the accumulated ``content``: "The - # capital is " is 15 chars, "Paris" is 5. - assert cite.start == 15 - assert cite.end == 20 - assert len(cite.sources) == 1 - assert cite.sources[0].id == "doc-paris" - assert cite.sources[0].type == "document" - - # -- 3. The citation's ``(start, end)`` offsets point into - # bytes that were already emitted as content-deltas earlier - # in the stream -- i.e. the citation is anchored to real - # already-shipped content. - cite_delta_idx = deltas.index(cite_delta) - earlier_content = "".join( - d.content for d in deltas[:cite_delta_idx] if d.content is not None - ) - assert earlier_content[cite.start : cite.end] == "Paris" - - # Everything after the citation delta is the trailing period. - later_content = "".join( - d.content for d in deltas[cite_delta_idx + 1 :] if d.content is not None - ) - assert later_content == "." - - # -- 4. Reasoning and content are never mixed on the same - # delta: the parser flips modes on the ``<|END_THINKING|>`` - # boundary and never emits a delta with both fields set. - for d in deltas: - assert not (d.reasoning is not None and d.content is not None) diff --git a/tests/parser/cohere/test_structural_tags.py b/tests/parser/cohere/test_structural_tags.py deleted file mode 100644 index 149c5229a7..0000000000 --- a/tests/parser/cohere/test_structural_tags.py +++ /dev/null @@ -1,476 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""``adjust_request``: folding tools and JSON schemas into Cohere structural tags.""" - -from __future__ import annotations - -import json -from collections import UserDict -from types import SimpleNamespace - -import pytest -from pydantic import ValidationError -from xgrammar.testing import _is_grammar_accept_string - -from vllm.entrypoints.generate.base.protocol import ( - JsonSchemaResponseFormat, - ResponseFormat, - StructuralTagResponseFormat, -) -from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest -from vllm.entrypoints.openai.responses.protocol import ResponsesRequest -from vllm.exceptions import VLLMValidationError -from vllm.parser.cohere_command import ( - CohereCommandParser, - CohereNormalizedTool, - _has_effective_tools, - _response_format_type, - _schema_dict_from_structured_outputs, - collect_tool_schema, - convert_schema_to_structural_tags, -) -from vllm.sampling_params import StructuredOutputsParams - -from .utils import MockCohereTokenizer, make_parser - -SCHEMA_A = {"type": "object", "properties": {"a": {"type": "string"}}} -SCHEMA_B = {"type": "object", "properties": {"b": {"type": "number"}}} -GET_WEATHER_TOOL = { - "type": "function", - "function": { - "name": "get_weather", - "parameters": { - "type": "object", - "properties": {"city": {"type": "string"}}, - }, - }, -} -NESTED_DEFS_PARAMETERS = { - "$defs": { - "Location": { - "type": "object", - "properties": { - "city": {"type": "string"}, - "country": {"type": "string"}, - }, - "required": ["city"], - } - }, - "type": "object", - "properties": {"location": {"$ref": "#/$defs/Location"}}, - "required": ["location"], -} -NESTED_DEFS_TOOL = { - "type": "function", - "function": {"name": "get_weather", "parameters": NESTED_DEFS_PARAMETERS}, -} -RECURSIVE_DEFS_PARAMETERS = { - "$defs": { - "Finding": { - "type": "object", - "properties": { - "note": {"type": "string"}, - "related": {"type": "array", "items": {"$ref": "#/$defs/Finding"}}, - }, - } - }, - "type": "object", - "properties": {"finding": {"$ref": "#/$defs/Finding"}}, -} -DRAFT07_DEFINITIONS_PARAMETERS = { - "definitions": { - "Location": { - "type": "object", - "properties": {"city": {"type": "string"}}, - } - }, - "type": "object", - "properties": {"location": {"$ref": "#/definitions/Location"}}, -} -VALID_STRUCTURAL_TAG = { - "type": "structural_tag", - "format": { - "type": "triggered_tags", - "tags": [ - { - "begin": "", - "content": {"type": "any_text"}, - "end": "", - } - ], - "triggers": [""], - }, -} - - -def _model_config(arch: str) -> SimpleNamespace: - return SimpleNamespace( - architecture=arch, - architectures=[arch], - hf_text_config=SimpleNamespace(architectures=[arch]), - ) - - -def _make_chat_request(**kwargs) -> ChatCompletionRequest: - data = {"messages": [{"role": "user", "content": "hi"}], "model": "m"} - data.update(kwargs) - return ChatCompletionRequest.model_validate(data) - - -def _first_json_schema(tag_json: str) -> dict | None: - outer = json.loads(tag_json) - for t in (outer.get("format") or {}).get("tags") or []: - c = t.get("content") or {} - if c.get("type") == "json_schema": - js = c.get("json_schema") - return js if isinstance(js, dict) else None - return None - - -def _content_types(tag_json: str) -> set[str]: - outer = json.loads(tag_json) - out: set[str] = set() - for t in (outer.get("format") or {}).get("tags") or []: - ty = (t.get("content") or {}).get("type") - if isinstance(ty, str): - out.add(ty) - return out - - -@pytest.fixture(scope="module") -def parser(tokenizer: MockCohereTokenizer) -> CohereCommandParser: - """Parser configured with a supported Cohere architecture.""" - return make_parser( - tokenizer, "cohere_command4", model_config=_model_config("Cohere2ForCausalLM") - ) - - -@pytest.fixture(scope="module") -def parser_no_model_config(tokenizer: MockCohereTokenizer) -> CohereCommandParser: - """Parser with no ``model_config`` (cannot resolve architecture).""" - return make_parser(tokenizer, "cohere_command4") - - -@pytest.fixture(scope="module") -def parser_unsupported_arch(tokenizer: MockCohereTokenizer) -> CohereCommandParser: - """Parser configured with an architecture that has no structural tag style.""" - return make_parser( - tokenizer, "cohere_command4", model_config=_model_config("LlamaForCausalLM") - ) - - -class TestAdjustRequestPassthrough: - def test_structured_outputs_structural_tag_not_modified(self, parser) -> None: - tag = json.dumps(VALID_STRUCTURAL_TAG) - r = _make_chat_request(structured_outputs={"structural_tag": tag}) - o = parser.adjust_request(r) - assert o.structured_outputs.structural_tag == tag - - def test_response_format_structural_tag_short_circuit(self, parser) -> None: - # ``ChatCompletionRequest`` validates ``response_format`` as a union; - # bare ``{"type": "structural_tag"}`` is invalid (use pydantic model). - rf = StructuralTagResponseFormat( - type="structural_tag", - format=VALID_STRUCTURAL_TAG["format"], - ) - r = _make_chat_request(response_format=rf) - o = parser.adjust_request(r) - assert _response_format_type(o.response_format) == "structural_tag" - assert o.structured_outputs is None - - -class TestAdjustRequestNoOp: - def test_no_schema_no_tools(self, parser) -> None: - o = parser.adjust_request(_make_chat_request()) - assert o.structured_outputs is None - assert o.response_format is None - - def test_no_model_config(self, parser_no_model_config) -> None: - inner = JsonSchemaResponseFormat(name="n", json_schema=SCHEMA_A) - r = _make_chat_request( - response_format=ResponseFormat(type="json_schema", json_schema=inner), - ) - o = parser_no_model_config.adjust_request(r) - assert o.response_format is not None - assert o.structured_outputs is None - - -class TestAdjustRequestUnsupportedArchitecture: - def test_json_schema_raises(self, parser_unsupported_arch) -> None: - inner = JsonSchemaResponseFormat(name="n", json_schema=SCHEMA_A) - r = _make_chat_request( - response_format=ResponseFormat(type="json_schema", json_schema=inner), - ) - with pytest.raises(ValueError, match="does not support"): - parser_unsupported_arch.adjust_request(r) - - -class TestAdjustRequestFoldFromResponseFormat: - @pytest.mark.parametrize( - "response_format, expected_schema", - [ - pytest.param( - ResponseFormat( - type="json_schema", - json_schema=JsonSchemaResponseFormat( - name="n", json_schema=SCHEMA_A - ), - ), - SCHEMA_A, - id="json_schema_pydantic", - ), - pytest.param( - { - "type": "json_schema", - "json_schema": {"name": "n", "schema": SCHEMA_A}, - }, - SCHEMA_A, - id="json_schema_dict", - ), - pytest.param( - {"type": "json_object"}, - {"type": "object"}, - id="json_object", - ), - ], - ) - def test_response_format_cleared( - self, parser, response_format, expected_schema - ) -> None: - r = _make_chat_request(response_format=response_format) - o = parser.adjust_request(r) - assert o.response_format is None - assert ( - _first_json_schema(o.structured_outputs.structural_tag) == expected_schema - ) - - -class TestHasEffectiveTools: - @pytest.mark.parametrize( - "tools, expected", - [ - pytest.param(None, False, id="none"), - pytest.param([], False, id="empty_list"), - pytest.param(" ", False, id="blank_str"), - pytest.param( - [{"type": "function", "function": {"name": "f"}}], - True, - id="non_empty_list", - ), - pytest.param('{"x": 1}', True, id="non_empty_str"), - ], - ) - def test_has_effective_tools(self, tools, expected) -> None: - assert _has_effective_tools(tools) is expected - - def test_convert_schema_json_only_with_empty_tools_list(self) -> None: - tag = convert_schema_to_structural_tags( - schema=SCHEMA_B, - tools=[], - model_architecture="Cohere2ForCausalLM", - ) - assert tag is not None - assert _first_json_schema(tag) == SCHEMA_B - - -class TestAdjustRequestFoldFromStructuredOutputs: - @pytest.mark.parametrize( - "structured_outputs, expected_schema", - [ - pytest.param({"json": SCHEMA_B}, SCHEMA_B, id="json_dict"), - pytest.param({"json": json.dumps(SCHEMA_B)}, SCHEMA_B, id="json_string"), - pytest.param( - {"json_object": True}, {"type": "object"}, id="json_object_flag" - ), - pytest.param( - StructuredOutputsParams(json=SCHEMA_B), - SCHEMA_B, - id="structured_outputs_dataclass", - ), - pytest.param( - {"json": {"name": "n", "schema": SCHEMA_A}}, - SCHEMA_A, - id="openai_wrapper_dict_unwrapped", - ), - ], - ) - def test_structured_outputs_folded( - self, parser, structured_outputs, expected_schema - ) -> None: - o = parser.adjust_request( - _make_chat_request(structured_outputs=structured_outputs), - ) - assert ( - _first_json_schema(o.structured_outputs.structural_tag) == expected_schema - ) - - def test_responses_request_default_empty_tools(self, parser) -> None: - """``ResponsesRequest.tools`` defaults to ``[]``, not ``None``.""" - r = ResponsesRequest.model_validate( - { - "input": "hi", - "model": "m", - "structured_outputs": {"json": SCHEMA_B}, - } - ) - assert r.tools == [] - o = parser.adjust_request(r) - assert _first_json_schema(o.structured_outputs.structural_tag) == SCHEMA_B - - def test_json_userdict_mapping_unwrapped(self) -> None: - inner = {"type": "object", "properties": {"u": {"type": "number"}}} - so = StructuredOutputsParams(json=UserDict(inner)) - assert _schema_dict_from_structured_outputs(so) == inner - - @pytest.mark.parametrize( - "json_value, match", - [ - pytest.param("{not json}", "valid JSON", id="invalid_json_string"), - pytest.param( - json.dumps(["a", "b"]), "JSON object", id="non_object_json_string" - ), - pytest.param(" ", "empty", id="empty_json_string"), - ], - ) - def test_structured_outputs_json_string_raises( - self, parser, json_value, match - ) -> None: - with pytest.raises(ValueError, match=match): - parser.adjust_request( - _make_chat_request(structured_outputs={"json": json_value}), - ) - - def test_deeply_nested_json_string_rejected(self, parser) -> None: - """json.loads raises RecursionError on a string nested this deeply.""" - schema = '{"type": "array", "items": ' * 20_000 + "{}" + "}" * 20_000 - with pytest.raises(VLLMValidationError, match="nested too deeply"): - parser.adjust_request( - _make_chat_request(structured_outputs={"json": schema}), - ) - - @pytest.mark.parametrize( - "construct", - [ - pytest.param( - lambda: _make_chat_request(structured_outputs={"json": [1, 2, 3]}), - id="chat_completion_request", - ), - pytest.param( - lambda: StructuredOutputsParams(json=[1, 2, 3]), - id="structured_outputs_params", - ), - ], - ) - def test_json_wrong_type_raises(self, construct) -> None: - """Non-str / non-dict ``json`` fails at Pydantic validation.""" - with pytest.raises(ValidationError): - construct() - - -class TestAdjustRequestPrecedence: - def test_response_format_over_structured_outputs_json(self, parser) -> None: - s_rf = {"type": "object", "properties": {"rf": {"type": "string"}}} - s_so = {"type": "object", "properties": {"so": {"type": "number"}}} - inner = JsonSchemaResponseFormat(name="n", json_schema=s_rf) - r = _make_chat_request( - response_format=ResponseFormat(type="json_schema", json_schema=inner), - structured_outputs={"json": s_so}, - ) - o = parser.adjust_request(r) - assert _first_json_schema(o.structured_outputs.structural_tag) == s_rf - - -class TestAdjustRequestTextPlusStructuredOutputs: - def test_text_response_format_preserved(self, parser) -> None: - sch = {"type": "object", "properties": {"k": {"type": "string"}}} - r = _make_chat_request( - response_format=ResponseFormat(type="text"), - structured_outputs={"json": sch}, - ) - o = parser.adjust_request(r) - assert o.response_format is not None - assert o.response_format.type == "text" - assert _first_json_schema(o.structured_outputs.structural_tag) == sch - - -class TestAdjustRequestTools: - def test_tools_only_command_a_grammar(self, parser) -> None: - o = parser.adjust_request( - _make_chat_request(tools=[GET_WEATHER_TOOL], tool_choice="auto"), - ) - assert "grammar" in _content_types(o.structured_outputs.structural_tag) - - def test_tools_plus_json_schema_both_kinds(self, parser) -> None: - inner = JsonSchemaResponseFormat( - name="n", - json_schema={"type": "object", "properties": {"r": {"type": "string"}}}, - ) - r = _make_chat_request( - response_format=ResponseFormat(type="json_schema", json_schema=inner), - tools=[GET_WEATHER_TOOL], - tool_choice="auto", - ) - o = parser.adjust_request(r) - types = _content_types(o.structured_outputs.structural_tag) - assert "grammar" in types - assert "json_schema" in types - - -class TestCollectToolSchemaDefs: - def test_nested_defs_tool_adjust_request(self, parser) -> None: - o = parser.adjust_request( - _make_chat_request(tools=[NESTED_DEFS_TOOL], tool_choice="auto"), - ) - assert "grammar" in _content_types(o.structured_outputs.structural_tag) - - @staticmethod - def _tool_calls_json(name: str, parameters: dict) -> str: - return json.dumps( - [{"tool_call_id": "0", "tool_name": name, "parameters": parameters}] - ) - - def test_recursive_defs(self) -> None: - grammar = collect_tool_schema( - [CohereNormalizedTool(name="report", parameters=RECURSIVE_DEFS_PARAMETERS)] - ) - nested = {"note": "a", "related": [{"note": "b", "related": []}]} - assert _is_grammar_accept_string( - grammar, self._tool_calls_json("report", {"finding": nested}) - ) - bad = {"note": 1, "related": []} - assert not _is_grammar_accept_string( - grammar, self._tool_calls_json("report", {"finding": bad}) - ) - - def test_draft07_definitions(self) -> None: - grammar = collect_tool_schema( - [ - CohereNormalizedTool( - name="get_weather", parameters=DRAFT07_DEFINITIONS_PARAMETERS - ) - ] - ) - assert _is_grammar_accept_string( - grammar, - self._tool_calls_json("get_weather", {"location": {"city": "Prague"}}), - ) - assert not _is_grammar_accept_string( - grammar, - self._tool_calls_json("get_weather", {"location": {"city": 1}}), - ) - - def test_deeply_nested_parameters_rejected(self) -> None: - """Tool schemas are compiled here, before request validation, and end up - inside an EBNF grammar that the nesting check cannot see later.""" - parameters = json.loads('{"type": "array", "items": ' * 200 + "{}" + "}" * 200) - with pytest.raises(VLLMValidationError, match="nested too deeply"): - collect_tool_schema( - [CohereNormalizedTool(name="deep", parameters=parameters)] - ) - - def test_input_parameters_not_mutated(self) -> None: - params = json.loads(json.dumps(NESTED_DEFS_PARAMETERS)) - collect_tool_schema( - [CohereNormalizedTool(name="get_weather", parameters=params)] - ) - assert params == NESTED_DEFS_PARAMETERS diff --git a/tests/parser/engine/test_delegating_replay.py b/tests/parser/engine/test_delegating_replay.py deleted file mode 100644 index 5cd48dd870..0000000000 --- a/tests/parser/engine/test_delegating_replay.py +++ /dev/null @@ -1,211 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Replay tests for DelegatingParser with engine adapters. - -Exercises DelegatingParser in engine-adapter mode to verify that delegated -routing produces correct output across chunk sizes. -See test_replay.py for tests that target engine parsers directly. - -Parser discovery is automatic: any engine parser in ``registered_adapters`` -that has both tool and reasoning adapters and a builder in -``trace_builder._BUILDERS`` is picked up with zero manual wiring. -""" - -from __future__ import annotations - -from typing import NamedTuple - -import pytest -from pydantic import TypeAdapter - -from tests.parser.engine.replay_harness import ( - CHUNK_SIZES, - DUMMY_TOOLS, - MockTokenizer, - _test_request, - assert_no_terminal_leakage, - assert_parse_output, - collect_output, - make_mock_tokenizer, - parse_non_streaming, - replay_streaming, -) -from tests.parser.engine.trace_builder import _BUILDERS, build_samples -from vllm.entrypoints.openai.chat_completion.protocol import ( - ChatCompletionToolsParam, -) -from vllm.parser.abstract_parser import DelegatingParser, Parser -from vllm.parser.engine import registered_adapters as _adapters_mod -from vllm.parser.engine.adapters import ( - ParserEngineReasoningAdapter, - ParserEngineToolAdapter, -) -from vllm.parser.mistral import MistralParser - -_TOOLS_VALIDATOR = TypeAdapter(list[ChatCompletionToolsParam]) - -# ── Pairing discovery ──────────────────────────────────────────────── - - -class _PairingInfo(NamedTuple): - parser_cls: type[Parser] - name: str - samples: tuple - - -def _discover_pairings() -> list[_PairingInfo]: - """Discover valid delegating pairings from registered engine adapters. - - Groups tool and reasoning adapters by their engine class, then builds - a DelegatingParser subclass for each engine that has both adapters - and a test builder. - """ - bare_tok = MockTokenizer(vocab={}, tokens=[]) - engines: dict[type, dict[str, type]] = {} - for obj in vars(_adapters_mod).values(): - if not isinstance(obj, type): - continue - if ( - issubclass(obj, ParserEngineToolAdapter) - and obj is not ParserEngineToolAdapter - ): - tool_adapter: type[ParserEngineToolAdapter] = obj - engines.setdefault(tool_adapter._parser_engine_cls, {})["tool"] = obj - elif ( - issubclass(obj, ParserEngineReasoningAdapter) - and obj is not ParserEngineReasoningAdapter - ): - reasoning_adapter: type[ParserEngineReasoningAdapter] = obj - engines.setdefault(reasoning_adapter._parser_engine_cls, {})[ - "reasoning" - ] = obj - - found: list[_PairingInfo] = [] - missing_builders: list[str] = [] - for engine_cls, adapters in engines.items(): - if "tool" not in adapters or "reasoning" not in adapters: - continue - if engine_cls is MistralParser: - # Mistral uses brace-balanced JSON tool args with no TOOL_END - # token, so it does not fit this TOOL_END-based replay harness. - # It is covered by tests/parser/mistral/ instead. - continue - cfg = engine_cls(bare_tok, None).parser_engine_config - if cfg.name not in _BUILDERS: - missing_builders.append(f"{engine_cls.__name__} (config.name={cfg.name!r})") - continue - if cfg.name == "inkling": - # Inkling uses typed structural blocks and opts out of token-id - # terminal matching; combined-parser replay coverage lives in - # test_inkling.py. - continue - if cfg.name == "granite": - # Granite has a JSON-array tool body with no TOOL_END terminal; - # its replay coverage lives in test_granite.py. - continue - - parser_cls = type( - f"_Delegating{engine_cls.__name__}", - (DelegatingParser,), - { - "reasoning_parser_cls": adapters["reasoning"], - "tool_parser_cls": adapters["tool"], - }, - ) - found.append( - _PairingInfo( - parser_cls=parser_cls, - name=cfg.name, - samples=build_samples(cfg.name), - ) - ) - if missing_builders: - raise RuntimeError( - f"Engine adapters in registered_adapters have no test builder " - f"in trace_builder._BUILDERS: {', '.join(missing_builders)}. " - f"Add a builder to _BUILDERS for each new parser." - ) - found.sort(key=lambda p: p.name) - return found - - -_PAIRINGS = _discover_pairings() - -_ALL_SAMPLES = [(p.parser_cls, s) for p in _PAIRINGS for s in p.samples] - - -@pytest.mark.parametrize("chunk_size", CHUNK_SIZES, ids=lambda c: f"chunk={c}") -@pytest.mark.parametrize( - "parser_cls,sample", - _ALL_SAMPLES, - ids=lambda v: v.id if hasattr(v, "id") else "", -) -def test_delegating_replay(parser_cls, sample, chunk_size): - tokenizer = make_mock_tokenizer(sample) - validated_tools = ( - _TOOLS_VALIDATOR.validate_python(sample.tools) if sample.tools else None - ) - parser = parser_cls( - tokenizer, - validated_tools, - chat_template_kwargs=sample.chat_template_kwargs, - ) - - deltas = replay_streaming( - parser, - sample.tokens, - chunk_size=chunk_size, - finished_on_last=True, - tools=sample.tools, - prompt_token_ids=sample.prompt_token_ids, - ) - output = collect_output(deltas) - assert_parse_output(output, sample) - - -_TOOL_CALL_SAMPLES = [ - (p.parser_cls, p.name, s) - for p in _PAIRINGS - for s in p.samples - if s.expected_tool_calls -] - - -@pytest.mark.parametrize( - "parser_cls,parser_name,sample", - _TOOL_CALL_SAMPLES, - ids=lambda v: v.id if hasattr(v, "id") else "", -) -def test_delegating_parse_tool_choice_none(parser_cls, parser_name, sample): - """Non-streaming parse() with tool_choice='none' via DelegatingParser - must not leak special tokens into content.""" - tokenizer = make_mock_tokenizer(sample) - validated_tools = ( - _TOOLS_VALIDATOR.validate_python(sample.tools) if sample.tools else None - ) - parser = parser_cls( - tokenizer, - validated_tools, - chat_template_kwargs=sample.chat_template_kwargs, - ) - - request = _test_request(tools=DUMMY_TOOLS) - request.tool_choice = "none" - - output = parse_non_streaming(parser, sample, request) - - assert output.tool_calls == [], ( - f"Expected no tool calls but got {output.tool_calls}" - ) - - cfg = parser._tool_parser._parser_engine.parser_engine_config - terminals = sorted( - v - for v in cfg.terminal_literals | set(cfg.token_id_terminals.values()) - if len(v) > 1 - ) - assert_no_terminal_leakage( - output, - terminals, - context=f"parser={parser_name}", - ) diff --git a/tests/parser/engine/test_replay.py b/tests/parser/engine/test_replay.py deleted file mode 100644 index e6339f8959..0000000000 --- a/tests/parser/engine/test_replay.py +++ /dev/null @@ -1,632 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Replay tests for engine parsers (holdback, skip-tool-parsing, adapters). - -Replays dynamically built token sequences at different chunk sizes and -holdback depths to verify chunk-size invariance and terminal-token hygiene. - -Parser discovery is automatic: any ``ParserEngine`` subclass registered in -``registered_adapters`` that also has a builder in ``trace_builder._BUILDERS`` -is picked up with zero manual wiring. -""" - -from __future__ import annotations - -import dataclasses -from typing import NamedTuple - -import pytest - -from tests.parser.engine.replay_harness import ( - DUMMY_TOOLS, - MockTokenizer, - Sample, - _test_request, - assert_no_terminal_leakage, - assert_parse_output, - collect_output, - make_mock_tokenizer, - parse_non_streaming, - replay_streaming, - replay_with_text_holdback, -) -from tests.parser.engine.trace_builder import _BUILDERS, build_samples -from vllm.parser.engine import registered_adapters as _adapters_mod -from vllm.parser.engine.parser_engine import ParserEngine -from vllm.parser.engine.parser_engine_config import ParserState -from vllm.parser.mistral import MistralParser - -# ── Parser discovery ───────────────────────────────────────────────── - - -class _ParserInfo(NamedTuple): - parser_cls: type[ParserEngine] - name: str - samples: tuple - terminals: list[str] - tool_end: str - think_end: str - tool_start: str - - -def _discover_parsers() -> list[_ParserInfo]: - """Discover engine parsers from registered_adapters that have test builders. - - Returns one ``_ParserInfo`` per parser, sorted by config name. - Raises ``RuntimeError`` if any registered parser lacks a builder. - """ - bare_tok = MockTokenizer(vocab={}, tokens=[]) - found: list[_ParserInfo] = [] - missing_builders: list[str] = [] - for obj in vars(_adapters_mod).values(): - if not ( - isinstance(obj, type) - and issubclass(obj, ParserEngine) - and obj is not ParserEngine - ): - continue - if obj is MistralParser: - # Mistral uses brace-balanced JSON tool args with no TOOL_END - # token, so it does not fit this TOOL_END-based replay harness. - # It is covered by tests/parser/mistral/ instead. - continue - cfg = obj(bare_tok, None).parser_engine_config - if cfg.name not in _BUILDERS: - missing_builders.append(f"{obj.__name__} (config.name={cfg.name!r})") - continue - if cfg.name in ("inkling", "plamo3", "minimax_m3"): - # Inkling uses typed structural blocks. PLaMo and MiniMax M3 - # markers can span several token IDs. None fits this TOOL_END-token - # harness; their direct and delegating coverage lives in dedicated - # tests. - continue - if cfg.name == "granite": - # Granite has a JSON-array tool body with no TOOL_END terminal, so - # it does not fit this token-terminal harness; its replay coverage - # lives in test_granite.py. - continue - tool_end = cfg.token_id_terminals.get("TOOL_END") - if not tool_end: - raise RuntimeError( - f"{obj.__name__} config missing 'TOOL_END' in token_id_terminals" - ) - all_vals = cfg.terminal_literals | set(cfg.token_id_terminals.values()) - found.append( - _ParserInfo( - parser_cls=obj, - name=cfg.name, - samples=build_samples(cfg.name), - terminals=sorted(v for v in all_vals if len(v) > 1), - tool_end=tool_end, - think_end=cfg.terminal_literal("THINK_END") or "", - tool_start=( - cfg.terminal_literal("TOOL_SECTION_START") - if (ParserState.CONTENT, "TOOL_SECTION_START") in cfg.transitions - else cfg.terminal_literal("TOOL_START") - ) - or "", - ) - ) - if missing_builders: - raise RuntimeError( - f"Engine parsers in registered_adapters have no test builder " - f"in trace_builder._BUILDERS: {', '.join(missing_builders)}. " - f"Add a builder to _BUILDERS for each new parser." - ) - found.sort(key=lambda p: p.name) - return found - - -_PARSERS = _discover_parsers() - - -def _make_parser(parser_cls: type[ParserEngine], tokenizer, sample: Sample, **extra): - kwargs = dict(extra) - if sample.chat_template_kwargs: - kwargs["chat_template_kwargs"] = sample.chat_template_kwargs - return parser_cls(tokenizer, sample.tools, **kwargs) - - -_ENGINE_PARSERS: dict[str, type[ParserEngine]] = { - f"{p.name}_engine": p.parser_cls for p in _PARSERS -} - -# ── Parametrize sample lists ───────────────────────────────────────── - -HOLDBACK_CONFIGS = [6, 12, 24] - -_REPLAY_SAMPLES = [(p.parser_cls, s, p.terminals) for p in _PARSERS for s in p.samples] - - -@pytest.mark.parametrize("holdback", HOLDBACK_CONFIGS, ids=lambda h: f"holdback{h}") -@pytest.mark.parametrize("chunk_size", [3, 5, 10], ids=lambda c: f"chunk{c}") -@pytest.mark.parametrize( - "parser_cls,sample,terminals", - _REPLAY_SAMPLES, - ids=lambda v: v.id if hasattr(v, "id") else "", -) -class TestReplayWithHoldback: - """Replay all parsers with simulated detokenizer holdback.""" - - def test_replay(self, parser_cls, sample, terminals, chunk_size, holdback): - tokenizer = make_mock_tokenizer(sample) - parser = _make_parser(parser_cls, tokenizer, sample) - deltas = replay_streaming( - parser, - sample.tokens, - chunk_size=chunk_size, - holdback_chars=holdback, - prompt_token_ids=sample.prompt_token_ids, - ) - output = collect_output(deltas) - - assert_parse_output(output, sample) - assert_no_terminal_leakage( - output, - terminals, - context=f"chunk_size={chunk_size}, holdback={holdback}", - ) - - -TEXT_HOLDBACK_DELAYS = [1, 2, 3] - - -@pytest.mark.parametrize("delay", TEXT_HOLDBACK_DELAYS, ids=lambda d: f"delay{d}") -@pytest.mark.parametrize( - "parser_cls,sample,terminals", - _REPLAY_SAMPLES, - ids=lambda v: v.id if hasattr(v, "id") else "", -) -class TestTextHoldback: - """Replay with production-like text/token-ID misalignment. - - In production the detokenizer sends token IDs immediately but holds - back text by N tokens. This exercises the TokenIDScanner deferred - terminal path that aligned-holdback tests do not cover. - """ - - def test_replay(self, parser_cls, sample, terminals, delay): - tokenizer = make_mock_tokenizer(sample) - parser = _make_parser(parser_cls, tokenizer, sample) - deltas = replay_with_text_holdback( - parser, - sample.tokens, - text_delay=delay, - prompt_token_ids=sample.prompt_token_ids, - ) - output = collect_output(deltas) - - assert_parse_output(output, sample) - assert_no_terminal_leakage( - output, - terminals, - context=f"text_delay={delay}", - ) - - -@pytest.mark.parametrize( - "chunk_size", [1, 2, 3, 5, 10, 19, 20, None], ids=lambda c: f"chunk{c}" -) -@pytest.mark.parametrize( - "parser_cls,sample,terminals", - _REPLAY_SAMPLES, - ids=lambda v: v.id if hasattr(v, "id") else "", -) -class TestReplay: - """Replay all parsers at varied chunk sizes without holdback.""" - - def test_replay(self, parser_cls, sample, terminals, chunk_size): - tokenizer = make_mock_tokenizer(sample) - parser = _make_parser(parser_cls, tokenizer, sample) - deltas = replay_streaming( - parser, - sample.tokens, - chunk_size=chunk_size, - prompt_token_ids=sample.prompt_token_ids, - ) - output = collect_output(deltas) - - assert_parse_output(output, sample) - assert_no_terminal_leakage(output, terminals) - - -_DEFERRAL_SAMPLES = [ - (p.parser_cls, s, p.tool_end) - for p in _PARSERS - for s in p.samples - if s.expected_tool_calls -] - - -@pytest.mark.parametrize( - "parser_cls,sample,tool_end_text", - _DEFERRAL_SAMPLES, - ids=lambda v: v.id if hasattr(v, "id") else getattr(v, "__name__", ""), -) -class TestDeferralFinish: - """Test that parse_delta(finished=True) resolves deferred scanner state. - - Simulates a production failure where delta_text is missing the - tool-call-end text but delta_token_ids has the token, causing the - scanner to defer it. Without finish(), the deferred state is lost - and tool call arguments are empty. - """ - - def test_misaligned_last_delta_with_finish(self, parser_cls, sample, tool_end_text): - tokenizer = make_mock_tokenizer(sample) - parser = _make_parser(parser_cls, tokenizer, sample) - - request = _test_request() - - all_ids = [tid for tid, _ in sample.tokens] - all_texts = [text for _, text in sample.tokens] - - tool_end_id = sample.vocab.get(tool_end_text) - split_idx = None - for i in range(len(all_ids) - 1, -1, -1): - if all_ids[i] == tool_end_id: - split_idx = i - break - - if split_idx is None: - pytest.skip(f"no {tool_end_text} token found") - - first_ids = all_ids[:split_idx] - first_text = "".join(all_texts[:split_idx]) - - last_ids = all_ids[split_idx:] - last_text_missing = "".join(all_texts[split_idx:]).replace(tool_end_text, "") - - result1 = parser.parse_delta( - first_text, - first_ids, - request, - prompt_token_ids=[], - finished=False, - ) - result2 = parser.parse_delta( - last_text_missing, last_ids, request, finished=True - ) - - output = collect_output([result1, result2]) - - tool_calls_only = dataclasses.replace( - sample, expected_reasoning=None, expected_content=None - ) - assert_parse_output(output, tool_calls_only) - - -@pytest.mark.parametrize( - "parser_cls,sample", - [(p.parser_cls, p.samples[0]) for p in _PARSERS], - ids=[p.name for p in _PARSERS], -) -class TestParserEngineAdjustRequest: - """Verify ParserEngine and its adapters set skip_special_tokens=False.""" - - def test_adjust_request_disables_skip_special_tokens(self, parser_cls, sample): - tokenizer = make_mock_tokenizer(sample) - parser = parser_cls(tokenizer, sample.tools) - request = _test_request() - assert request.skip_special_tokens is True - adjusted = parser.adjust_request(request) - assert adjusted.skip_special_tokens is False - - -_TOOL_CALL_SAMPLES = [ - (p.parser_cls, s, p.think_end, p.tool_start) - for p in _PARSERS - for s in p.samples - if s.expected_tool_calls - and s.expected_reasoning - # Inkling's typed-block format (structural role/kind tokens, shared - # block-end token) doesn't fit the generic reasoning/tool split - # below; its filtering modes are covered in test_inkling.py instead. - and p.name != "inkling" -] - - -def _tool_suppression_expectations( - sample, think_end: str, tool_start: str, *, include_tool_block: bool -) -> tuple[str, str]: - reasoning, content = _raw_tool_suppression_expectations( - sample, think_end, tool_start, include_tool_block=include_tool_block - ) - if sample.content_lstrip: - content = content.lstrip(sample.content_lstrip) - return reasoning, content - - -def _raw_tool_suppression_expectations( - sample, think_end: str, tool_start: str, *, include_tool_block: bool -) -> tuple[str, str]: - """Expected (reasoning, content) from the raw token text. - - With ``include_tool_block=True`` (skip_tool_parsing / reasoning - adapter first pass), tool terminal text is preserved as content so - a second-pass parser can see it. - - With ``include_tool_block=False`` (_suppress_tool_calls / - tool_choice='none'), the state machine consumes tool blocks and - only non-tool content survives. - """ - full_text = "".join(text for _, text in sample.tokens) - reasoning = sample.expected_reasoning - idx = full_text.find(reasoning) - if idx < 0: - return (full_text, "") - after_reasoning = full_text[idx + len(reasoning) :] - if think_end: - pos = after_reasoning.find(think_end) - if pos >= 0: - if include_tool_block: - return (reasoning, after_reasoning[pos + len(think_end) :]) - after_reasoning = after_reasoning[pos + len(think_end) :] - if tool_start: - pos = after_reasoning.find(tool_start) - if pos >= 0: - if include_tool_block: - return (reasoning, after_reasoning[pos:]) - return (reasoning, after_reasoning[:pos]) - if include_tool_block: - return (full_text, "") - return (reasoning, after_reasoning) - - -@pytest.mark.parametrize("chunk_size", [1, 5, None], ids=lambda c: f"chunk{c}") -@pytest.mark.parametrize( - "mode", - ["skip_tool_parsing", "suppress_tool_calls"], - ids=["skip_tool_parsing", "suppress_tool_calls"], -) -@pytest.mark.parametrize( - "parser_cls,sample,think_end,tool_start", - _TOOL_CALL_SAMPLES, - ids=lambda v: v.id if hasattr(v, "id") else getattr(v, "__name__", ""), -) -class TestToolCallFilteringReplay: - """Replay with tool calls not extracted, in both filtering modes. - - ``skip_tool_parsing`` (reasoning adapter first pass): tool terminal - text is preserved as content for a second-pass tool parser. - - ``suppress_tool_calls`` (tool_choice='none'): tool call blocks are - consumed by the state machine and do not leak into content. - """ - - def test_replay(self, parser_cls, sample, think_end, tool_start, mode, chunk_size): - tokenizer = make_mock_tokenizer(sample) - kwargs = {} - if sample.chat_template_kwargs: - kwargs["chat_template_kwargs"] = sample.chat_template_kwargs - parser = parser_cls(tokenizer, **kwargs) - - request = _test_request() - request.tools = DUMMY_TOOLS - if mode == "skip_tool_parsing": - parser.skip_tool_parsing = True - else: - request.tool_choice = "none" - - all_ids = [tid for tid, _ in sample.tokens] - all_texts = [text for _, text in sample.tokens] - if chunk_size is None: - chunk_size = len(all_ids) - - results = [] - chunks = list(range(0, len(all_ids), chunk_size)) - for i, start in enumerate(chunks): - end = min(start + chunk_size, len(all_ids)) - is_last = i == len(chunks) - 1 - result = parser.parse_delta( - "".join(all_texts[start:end]), - all_ids[start:end], - request, - prompt_token_ids=(sample.prompt_token_ids or []) - if start == 0 - else None, - finished=is_last, - ) - results.append(result) - - output = collect_output(results) - - include_block = mode == "skip_tool_parsing" - expected_reasoning, expected_content = _tool_suppression_expectations( - sample, think_end, tool_start, include_tool_block=include_block - ) - - assert output.reasoning == expected_reasoning, ( - f"Reasoning mismatch (mode={mode}):\n" - f" expected: {expected_reasoning!r}\n" - f" actual: {output.reasoning!r}" - ) - assert output.tool_calls == [], ( - f"Expected no tool calls (mode={mode}) but got {output.tool_calls}" - ) - assert output.content == expected_content, ( - f"Content mismatch (mode={mode}):\n" - f" expected: {expected_content!r}\n" - f" actual: {output.content!r}" - ) - - -@pytest.mark.parametrize( - "parser_cls,sample,think_end,tool_start", - _TOOL_CALL_SAMPLES, - ids=lambda v: v.id if hasattr(v, "id") else getattr(v, "__name__", ""), -) -class TestToolCallFilteringNonStreaming: - """Non-streaming parse() with tool_choice='none' must suppress tool - calls and not leak special tokens into content.""" - - def test_parse(self, parser_cls, sample, think_end, tool_start): - tokenizer = make_mock_tokenizer(sample) - kwargs = {} - if sample.chat_template_kwargs: - kwargs["chat_template_kwargs"] = sample.chat_template_kwargs - parser = parser_cls(tokenizer, **kwargs) - - request = _test_request() - request.tools = DUMMY_TOOLS - request.tool_choice = "none" - - output = parse_non_streaming(parser, sample, request) - - expected_reasoning, expected_content = _tool_suppression_expectations( - sample, think_end, tool_start, include_tool_block=False - ) - assert output.reasoning == expected_reasoning, ( - f"Reasoning mismatch:\n" - f" expected: {expected_reasoning!r}\n" - f" actual: {output.reasoning!r}" - ) - assert output.tool_calls == [], ( - f"Expected no tool calls but got {output.tool_calls}" - ) - assert output.content == expected_content, ( - f"Content mismatch:\n" - f" expected: {expected_content!r}\n" - f" actual: {output.content!r}" - ) - - -_WS_TOOL_SAMPLES = [(t[0], t[1]) for t in _TOOL_CALL_SAMPLES if "whitespace" in t[1].id] - - -@pytest.mark.parametrize( - "parser_cls,sample", - _WS_TOOL_SAMPLES, - ids=lambda v: v.id if hasattr(v, "id") else getattr(v, "__name__", ""), -) -class TestToolChoiceNoneStreamingParity: - """Streaming and non-streaming must return the same content - when tool_choice='none' suppresses tool calls.""" - - def test_content_matches(self, parser_cls, sample): - tokenizer = make_mock_tokenizer(sample) - kwargs = {} - if sample.chat_template_kwargs: - kwargs["chat_template_kwargs"] = sample.chat_template_kwargs - request = _test_request() - request.tools = DUMMY_TOOLS - request.tool_choice = "none" - - ns_output = parse_non_streaming( - parser_cls(tokenizer, **kwargs), - sample, - request, - ) - - s_parser = parser_cls(tokenizer, **kwargs) - results = [] - for i, (tid, text) in enumerate(sample.tokens): - is_last = i == len(sample.tokens) - 1 - results.append( - s_parser.parse_delta( - text, - [tid], - request, - prompt_token_ids=(sample.prompt_token_ids or []) - if i == 0 - else None, - finished=is_last, - ) - ) - s_output = collect_output(results) - - assert ns_output.content == s_output.content, ( - f"Streaming/non-streaming content mismatch:\n" - f" streaming: {s_output.content!r}\n" - f" non-streaming: {ns_output.content!r}" - ) - - -_DROP_TOKENS = {"": 99990, "": 99991} - - -def _inject_drop_tokens(sample): - """Insert at stream start and between the first two tokens.""" - new_vocab = {**sample.vocab, **_DROP_TOKENS} - tokens = list(sample.tokens) - tokens.insert(0, (99990, "")) - if len(tokens) >= 3: - tokens.insert(2, (99991, "")) - else: - tokens.append((99991, "")) - return dataclasses.replace(sample, vocab=new_vocab, tokens=tokens) - - -class TestDropTokenReplay: - """Verify unconfigured special tokens are silently dropped across - all parsers and chunk sizes.""" - - @pytest.mark.parametrize( - "parser_info", - _PARSERS, - ids=[p.name for p in _PARSERS], - ) - @pytest.mark.parametrize("chunk_size", [1, 3, None]) - def test_drop_tokens_removed_from_output(self, parser_info, chunk_size): - for sample in parser_info.samples: - injected = _inject_drop_tokens(sample) - tokenizer = make_mock_tokenizer(injected) - parser = _make_parser(parser_info.parser_cls, tokenizer, sample) - - results = replay_streaming( - parser, - injected.tokens, - chunk_size=chunk_size, - tools=sample.tools, - prompt_token_ids=sample.prompt_token_ids, - ) - output = collect_output(results) - - assert_no_terminal_leakage( - output, - list(_DROP_TOKENS.keys()), - context=f"parser={parser_info.name}, chunk={chunk_size}", - ) - assert_parse_output(output, sample) - - -class TestDropTokenNonStreaming: - """Non-streaming parse() must also strip unconfigured special tokens.""" - - @pytest.mark.parametrize( - "parser_info", - _PARSERS, - ids=[p.name for p in _PARSERS], - ) - def test_drop_tokens_removed_from_output(self, parser_info): - for sample in parser_info.samples: - injected = _inject_drop_tokens(sample) - tokenizer = make_mock_tokenizer(injected) - parser = _make_parser(parser_info.parser_cls, tokenizer, sample) - - request = _test_request(tools=sample.tools) - output = parse_non_streaming(parser, injected, request) - - assert_no_terminal_leakage( - output, - list(_DROP_TOKENS.keys()), - context=f"parser={parser_info.name}", - ) - - -class TestAdapterReferences: - """Verify make_adapters sets reasoning/tool parser class refs on parser engine - parser classes so the serving layer finds them and calls adjust_request.""" - - @pytest.mark.parametrize( - "parser_name", - list(_ENGINE_PARSERS.keys()), - ) - def test_adapter_cls_refs_set(self, parser_name): - parser_cls = _ENGINE_PARSERS[parser_name] - assert parser_cls.reasoning_parser_cls is not None, ( - f"{parser_name}: reasoning_parser_cls is None" - ) - assert parser_cls.tool_parser_cls is not None, ( - f"{parser_name}: tool_parser_cls is None" - ) diff --git a/tests/parser/engine/test_ufffd_reasoning_transition.py b/tests/parser/engine/test_ufffd_reasoning_transition.py index ffd2ead95d..a1312c5657 100644 --- a/tests/parser/engine/test_ufffd_reasoning_transition.py +++ b/tests/parser/engine/test_ufffd_reasoning_transition.py @@ -23,8 +23,6 @@ from tests.parser.engine.replay_harness import ( ) from vllm.parser.abstract_parser import DelegatingParser from vllm.parser.engine.registered_adapters import ( - Glm47MoeParserReasoningAdapter, - Glm47MoeParserToolAdapter, Qwen3ParserReasoningAdapter, Qwen3ParserToolAdapter, ) @@ -62,11 +60,6 @@ class ByteFallbackMockTokenizer(MockTokenizer): # ── Model-specific DelegatingParser subclasses ─────────────────────── -class _Glm47Delegating(DelegatingParser): - reasoning_parser_cls = Glm47MoeParserReasoningAdapter - tool_parser_cls = Glm47MoeParserToolAdapter - - class _Qwen3Delegating(DelegatingParser): reasoning_parser_cls = Qwen3ParserReasoningAdapter tool_parser_cls = Qwen3ParserToolAdapter @@ -92,20 +85,6 @@ EXPECTED_REASONING = "Let me think about Samsung." EXPECTED_CONTENT = "삼성전자의 주가를 분석합니다." _MODEL_CONFIGS = [ - pytest.param( - { - "": 50, - "": 51, - "": 60, - "": 61, - "": 62, - "": 63, - "": 64, - "": 65, - }, - _Glm47Delegating, - id="glm47", - ), pytest.param( { "": 50, @@ -165,7 +144,7 @@ class TestUfffdReasoningTransition: vocab = dict(_MODEL_CONFIGS[0].values[0]) tokenizer = ByteFallbackMockTokenizer(vocab, tokens, ufffd_ids) - parser = _Glm47Delegating(tokenizer) + parser = _Qwen3Delegating(tokenizer) deltas = replay_streaming( parser, tokens, diff --git a/tests/parser/engine/trace_builder.py b/tests/parser/engine/trace_builder.py deleted file mode 100644 index 284ff7048d..0000000000 --- a/tests/parser/engine/trace_builder.py +++ /dev/null @@ -1,1358 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""On-demand trace builder for parser engine testing and benchmarks. - -Generates token sequences programmatically from model-agnostic scenario -definitions. Each model format handler knows how to render scenarios -into the model's output format, tokenize them with correct special token -IDs, and compute expected parse outputs. - -Every generated sample is self-validated by replaying it through the -real parser before being returned. -""" - -from __future__ import annotations - -import functools -import json -from collections.abc import Callable -from dataclasses import dataclass, replace -from typing import Any - -from tests.parser.engine.replay_harness import ( - MockTokenizer, - Sample, - assert_parse_output, - collect_output, - replay_streaming, -) -from vllm.entrypoints.openai.chat_completion.protocol import ( - ChatCompletionToolsParam, -) -from vllm.parser import plamo3 -from vllm.parser.engine.registered_adapters import ( - DeepSeekV4Parser, - DeepSeekV32Parser, - DeepSeekV41Parser, - Gemma4Parser, - Glm47MoeParser, - GraniteParser, - GraniteThinkingParser, - InklingParser, - KimiK2Parser, - MiMoParser, - MinimaxM2Parser, - MinimaxM3Parser, - NemotronV3Parser, - Plamo3Parser, - Qwen3Parser, - SeedOssParser, - Step3p5Parser, -) - -# ── Data structures ────────────────────────────────────────────────── - - -@dataclass -class ToolCallSpec: - name: str - arguments: dict[str, Any] - - -@dataclass -class Scenario: - id: str - description: str - reasoning: str | None = None - content: str | None = None - tool_calls: list[ToolCallSpec] | None = None - after_tool_response: bool = False - - -# ── Scenarios ──────────────────────────────────────────────────────── - -_READ_TOOL = ToolCallSpec("read_file", {"path": "/tmp/test.txt"}) -_BASH_TOOL = ToolCallSpec( - "bash", {"command": "hostname", "description": "Get hostname"} -) -_WEATHER_TOOL = ToolCallSpec( - "get_weather", - {"city": "Dallas", "state": "TX", "unit": "fahrenheit"}, -) -_COMPLEX_TOOL = ToolCallSpec( - "search", - { - "query": "vllm parser", - "filters": {"language": "python", "min_stars": 100}, - "tags": ["ml", "inference"], - "limit": 10, - "verbose": True, - }, -) - -SCENARIOS: list[Scenario] = [ - Scenario( - id="think-then-tool", - description="Reasoning then single tool call", - reasoning="Let me check the file.", - tool_calls=[_READ_TOOL], - ), - Scenario( - id="think-then-parallel-tools", - description="Reasoning then two parallel tool calls", - reasoning="I need to run both commands.", - tool_calls=[_BASH_TOOL, _WEATHER_TOOL], - ), - Scenario( - id="think-then-content", - description="Reasoning then content response", - reasoning="Let me think about this carefully.", - content="The answer is 42.", - ), - Scenario( - id="content-only", - description="Plain content response without reasoning", - content="Hello! How can I help you today?", - ), - Scenario( - id="tool-only", - description="Tool call without reasoning", - tool_calls=[_READ_TOOL], - ), - Scenario( - id="complex-json-args", - description="Tool call with nested objects, arrays, numbers, booleans", - reasoning="This needs a complex query.", - tool_calls=[_COMPLEX_TOOL], - ), - Scenario( - id="whitespace-before-tool", - description="Whitespace-only content before tool call", - content="\n\n", - tool_calls=[_WEATHER_TOOL], - ), - Scenario( - id="think-content-tool", - description="Reasoning, content, then tool call", - reasoning="Let me analyze and then fetch data.", - content="Checking the weather now.", - tool_calls=[_WEATHER_TOOL], - ), - Scenario( - id="think-whitespace-tool", - description="Reasoning, whitespace-only gap, then tool call", - reasoning="Let me check the file contents.", - content="\n\n", - tool_calls=[_READ_TOOL], - ), - Scenario( - id="empty-reasoning-content", - description="Empty reasoning section followed by content", - reasoning="", - content="The epoch timestamp is 1779111346.", - ), - Scenario( - id="tool-after-tool-response", - description="Tool call immediately after tool response (agentic flow)", - tool_calls=[_READ_TOOL], - after_tool_response=True, - ), - Scenario( - id="empty-tool-block", - description="Empty tool block followed by content (edge case recovery)", - content="Content after empty tools.", - tool_calls=[], - ), -] - - -# ── Tokenization ───────────────────────────────────────────────────── - - -def _word_split(text: str) -> list[str]: - """Split text into word-like tokens, preserving all characters.""" - if not text: - return [] - parts: list[str] = [] - current = "" - for ch in text: - if ch in " \t\n\r" and current and current[-1] not in " \t\n\r": - parts.append(current) - current = ch - else: - current += ch - if current: - parts.append(current) - return parts - - -def _tokenize( - segments: list[tuple[str, bool]], - vocab: dict[str, int], - start_id: int = 100, -) -> list[tuple[int, str]]: - """Build token list from segments. - - Each segment is ``(text, is_special)``. Special segments use vocab - IDs; content segments are word-split with sequential IDs. - """ - tokens: list[tuple[int, str]] = [] - next_id = start_id - - for text, is_special in segments: - if not text: - continue - if is_special: - tid = vocab.get(text) - if tid is None: - raise ValueError(f"Special token {text!r} not in vocab") - tokens.append((tid, text)) - else: - for word in _word_split(text): - tokens.append((next_id, word)) - next_id += 1 - - return tokens - - -# ── Tool definitions ───────────────────────────────────────────────── - - -def _infer_schema(value: object) -> dict: - """Infer a JSON Schema from a Python value, recursing into dicts/lists.""" - if isinstance(value, bool): - return {"type": "boolean"} - if isinstance(value, int): - return {"type": "integer"} - if isinstance(value, float): - return {"type": "number"} - if isinstance(value, str): - return {"type": "string"} - if isinstance(value, dict): - return { - "type": "object", - "properties": {k: _infer_schema(v) for k, v in value.items()}, - } - if isinstance(value, list) and value: - return {"type": "array", "items": _infer_schema(value[0])} - if isinstance(value, list): - return {"type": "array"} - return {} - - -def _tool_defs(tool_calls: list[ToolCallSpec]) -> list[dict]: - """Generate OpenAI-style tool definitions from tool call specs.""" - seen: set[str] = set() - tools: list[dict] = [] - for tc in tool_calls: - if tc.name in seen: - continue - seen.add(tc.name) - properties = {k: _infer_schema(v) for k, v in tc.arguments.items()} - tools.append( - { - "type": "function", - "function": { - "name": tc.name, - "parameters": { - "type": "object", - "properties": properties, - }, - }, - } - ) - return tools - - -# ── Format handlers ────────────────────────────────────────────────── - - -def _expected_tc(scenario: Scenario) -> list[dict] | None: - if not scenario.tool_calls: - return None - return [{"name": tc.name, "arguments": tc.arguments} for tc in scenario.tool_calls] - - -def _expected_tools(scenario: Scenario) -> list[dict] | None: - return _tool_defs(scenario.tool_calls) if scenario.tool_calls else None - - -def _validate_sample(sample: Sample, parser_cls: type, **kwargs) -> None: - """Replay sample through the real parser and assert correctness.""" - tokenizer = MockTokenizer(vocab=dict(sample.vocab), tokens=sample.tokens) - parser = parser_cls(tokenizer, sample.tools, **kwargs) - deltas = replay_streaming( - parser, - sample.tokens, - chunk_size=1, - tools=sample.tools, - prompt_token_ids=sample.prompt_token_ids, - ) - output = collect_output(deltas) - assert_parse_output(output, sample) - - -def _validate_tools( - tools: list[dict] | None, -) -> list[ChatCompletionToolsParam] | None: - if not tools: - return None - return [ChatCompletionToolsParam.model_validate(t) for t in tools] - - -def _make_sample( - sample_id: str, - description: str, - vocab: dict[str, int], - segments: list[tuple[str, bool]], - expected_reasoning: str | None, - expected_content: str | None, - expected_tool_calls: list[dict] | None, - tools: list[dict] | None, - chat_template_kwargs: dict | None = None, - prompt_token_ids: list[int] | None = None, -) -> Sample: - tokens = _tokenize(segments, vocab) - return Sample( - id=sample_id, - description=description, - source="trace-builder", - vocab=dict(vocab), - tokens=tokens, - expected_reasoning=expected_reasoning, - expected_content=expected_content, - expected_tool_calls=expected_tool_calls, - tools=_validate_tools(tools), - chat_template_kwargs=chat_template_kwargs, - prompt_token_ids=prompt_token_ids, - ) - - -# ── Qwen3 (XML tool format, starts in REASONING) ──────────────────── - -_QWEN3_VOCAB: dict[str, int] = { - "": 50, - "": 51, - "": 60, - "": 61, -} - - -def _qwen3_arg_value(value: Any) -> str: - if isinstance(value, bool): - return "true" if value else "false" - if isinstance(value, (int, float)): - return str(value) - if isinstance(value, str): - return value - return json.dumps(value, ensure_ascii=False) - - -def _qwen3_tool_segments(tc: ToolCallSpec) -> list[tuple[str, bool]]: - parts = [f"\n"] - for key, value in tc.arguments.items(): - parts.append(f"\n{_qwen3_arg_value(value)}") - parts.append("\n\n") - return [ - ("", True), - ("".join(parts), False), - ("", True), - ] - - -def _qwen3_segments( - scenario: Scenario, - tool_segments: Callable[[ToolCallSpec], list[tuple[str, bool]]] = ( - _qwen3_tool_segments - ), -) -> list[tuple[str, bool]]: - segs: list[tuple[str, bool]] = [] - if scenario.reasoning is not None: - segs.append((scenario.reasoning, False)) - if scenario.content is not None or scenario.tool_calls is not None: - segs.append(("", True)) - if scenario.tool_calls is not None and not scenario.tool_calls: - segs.append(("", True)) - segs.append(("", True)) - if scenario.content is not None: - segs.append((scenario.content, False)) - if scenario.tool_calls: - for tc in scenario.tool_calls: - segs.extend(tool_segments(tc)) - return segs - - -def _qwen3_expected_content(scenario: Scenario) -> str | None: - if ( - scenario.content is not None - and scenario.tool_calls - and not scenario.content.strip() - ): - return "" - return scenario.content - - -def _build_qwen3( - scenario: Scenario, - name: str = "qwen3", - parser_cls: type = Qwen3Parser, - strip_trailing_ws: bool = False, - validate: bool = True, - tool_segments: Callable[[ToolCallSpec], list[tuple[str, bool]]] = ( - _qwen3_tool_segments - ), -) -> Sample: - expected_reasoning: str | None - if scenario.reasoning is not None: - r = scenario.reasoning - if strip_trailing_ws: - r = r.rstrip() - expected_reasoning = r - else: - expected_reasoning = "" - - sample = _make_sample( - sample_id=f"{name}-{scenario.id}", - description=scenario.description, - vocab=_QWEN3_VOCAB, - segments=_qwen3_segments(scenario, tool_segments), - expected_reasoning=expected_reasoning, - expected_content=_qwen3_expected_content(scenario), - expected_tool_calls=_expected_tc(scenario), - tools=_expected_tools(scenario), - ) - if validate: - _validate_sample(sample, parser_cls) - return sample - - -# ── MiMo (compact Qwen3 XML, no newlines between tags) ─────────────── - - -def _mimo_tool_segments(tc: ToolCallSpec) -> list[tuple[str, bool]]: - parts = [f""] - for key, value in tc.arguments.items(): - parts.append(f"{_qwen3_arg_value(value)}") - parts.append("") - return [ - ("", True), - ("".join(parts), False), - ("", True), - ] - - -def _build_mimo(scenario: Scenario, validate: bool = True) -> Sample: - return _build_qwen3( - scenario, - name="mimo", - parser_cls=MiMoParser, - validate=validate, - tool_segments=_mimo_tool_segments, - ) - - -# ── Step-3.5 (Qwen3 XML, trailing reasoning whitespace stripped) ──── - - -def _build_step3p5(scenario: Scenario, validate: bool = True) -> Sample: - return _build_qwen3( - scenario, - name="step3p5", - parser_cls=Step3p5Parser, - strip_trailing_ws=True, - validate=validate, - ) - - -# ── MiniMax M2 (XML invoke format, starts in REASONING) ────────────── - -_MINIMAX_M2_VOCAB: dict[str, int] = { - "": 50, - "": 51, - "": 60, - "": 61, -} - - -def _minimax_m2_arg_value(value: Any) -> str: - if isinstance(value, bool): - return "true" if value else "false" - if isinstance(value, (int, float)): - return str(value) - if isinstance(value, str): - return value - return json.dumps(value, ensure_ascii=False) - - -def _minimax_m2_tool_segments(tool_calls: list[ToolCallSpec]) -> list[tuple[str, bool]]: - segs: list[tuple[str, bool]] = [("", True)] - for tc in tool_calls: - segs.append((f'', False)) - for key, value in tc.arguments.items(): - segs.append( - ( - f'' - f"{_minimax_m2_arg_value(value)}" - "", - False, - ) - ) - segs.append(("", False)) - segs.append(("", True)) - return segs - - -def _minimax_m2_segments(scenario: Scenario) -> list[tuple[str, bool]]: - segs: list[tuple[str, bool]] = [] - if scenario.reasoning is not None: - segs.append((scenario.reasoning, False)) - if scenario.content is not None or scenario.tool_calls is not None: - segs.append(("", True)) - if scenario.tool_calls is not None and not scenario.tool_calls: - segs.append(("", True)) - segs.append(("", True)) - if scenario.content is not None: - segs.append((scenario.content, False)) - if scenario.tool_calls: - segs.extend(_minimax_m2_tool_segments(scenario.tool_calls)) - return segs - - -def _build_minimax_m2(scenario: Scenario, validate: bool = True) -> Sample: - expected_reasoning: str | None - if scenario.reasoning is not None: - expected_reasoning = scenario.reasoning.rstrip() - else: - expected_reasoning = "" - - sample = _make_sample( - sample_id=f"minimax_m2-{scenario.id}", - description=scenario.description, - vocab=_MINIMAX_M2_VOCAB, - segments=_minimax_m2_segments(scenario), - expected_reasoning=expected_reasoning, - expected_content=_qwen3_expected_content(scenario), - expected_tool_calls=_expected_tc(scenario), - tools=_expected_tools(scenario), - ) - if validate: - _validate_sample(sample, MinimaxM2Parser) - return sample - - -# ── MiniMax M3 (namespaced XML elements, no reasoning) ─────────────── - -_MINIMAX_M3_NS = "]<]minimax[>[" - - -def _minimax_m3_value(value: Any) -> str: - """Render a value the way the M3 chat template's ``to_xml`` does.""" - ns = _MINIMAX_M3_NS - if isinstance(value, dict): - return "".join( - f"{ns}<{key}>{_minimax_m3_value(item)}{ns}" - for key, item in value.items() - ) - if isinstance(value, list): - return "".join( - f"{ns}{_minimax_m3_value(item)}{ns}" for item in value - ) - if isinstance(value, bool): - return "true" if value else "false" - return str(value) - - -def _minimax_m3_segments(scenario: Scenario) -> list[tuple[str, bool]]: - ns = _MINIMAX_M3_NS - segs: list[tuple[str, bool]] = [] - if scenario.content: - # The tool parser receives reasoning-stripped content. - segs.append((scenario.content, False)) - if scenario.tool_calls: - invokes = "".join( - f'{ns}' - f"{_minimax_m3_value(tc.arguments)}{ns}\n" - for tc in scenario.tool_calls - ) - segs.append((f"{ns}\n{invokes}{ns}", False)) - return segs - - -def _minimax_m3_expected_content(scenario: Scenario) -> str | None: - if scenario.tool_calls and not (scenario.content or "").strip(): - return None - return scenario.content - - -def _build_minimax_m3(scenario: Scenario, validate: bool = True) -> Sample: - sample = _make_sample( - sample_id=f"minimax_m3-{scenario.id}", - description=scenario.description, - vocab={}, - segments=_minimax_m3_segments(scenario), - expected_reasoning=None, - expected_content=_minimax_m3_expected_content(scenario), - expected_tool_calls=_expected_tc(scenario), - tools=_expected_tools(scenario), - ) - if validate: - _validate_sample(sample, MinimaxM3Parser) - return sample - - -# ── Gemma4 (channel reasoning, custom arg format) ──────────────────── - -_GEMMA4_VOCAB: dict[str, int] = { - "<|channel>": 50, - "": 51, - "<|tool_call>": 48, - "": 49, - '<|"|>': 52, - "<|turn>": 53, - "<|tool_response>": 54, -} -_GEMMA4_THOUGHT_PREFIX = "thought\n" -_GEMMA4_QUOTE = '<|"|>' - - -def _gemma4_value_segments(value: Any) -> list[tuple[str, bool]]: - """Render a value in Gemma4 arg format as segments.""" - if isinstance(value, str): - return [(_GEMMA4_QUOTE, True), (value, False), (_GEMMA4_QUOTE, True)] - if isinstance(value, bool): - return [("true" if value else "false", False)] - if isinstance(value, (int, float)): - return [(str(value), False)] - if isinstance(value, dict): - segs: list[tuple[str, bool]] = [("{", False)] - for i, (k, v) in enumerate(value.items()): - if i > 0: - segs.append((",", False)) - segs.append((f"{k}:", False)) - segs.extend(_gemma4_value_segments(v)) - segs.append(("}", False)) - return segs - if isinstance(value, list): - segs = [("[", False)] - for i, item in enumerate(value): - if i > 0: - segs.append((",", False)) - segs.extend(_gemma4_value_segments(item)) - segs.append(("]", False)) - return segs - return [(json.dumps(value, ensure_ascii=False), False)] - - -def _gemma4_tool_segments(tc: ToolCallSpec) -> list[tuple[str, bool]]: - segs: list[tuple[str, bool]] = [ - ("<|tool_call>", True), - (f"call:{tc.name}", False), - ("{", False), - ] - for i, (key, value) in enumerate(tc.arguments.items()): - if i > 0: - segs.append((",", False)) - segs.append((f"{key}:", False)) - segs.extend(_gemma4_value_segments(value)) - segs.append(("}", False)) - segs.append(("", True)) - return segs - - -def _gemma4_segments(scenario: Scenario) -> list[tuple[str, bool]]: - segs: list[tuple[str, bool]] = [] - if scenario.reasoning is not None: - segs.append(("<|channel>", True)) - segs.append((_GEMMA4_THOUGHT_PREFIX, False)) - segs.append((scenario.reasoning, False)) - segs.append(("", True)) - if scenario.tool_calls is not None and not scenario.tool_calls: - segs.append(("<|tool_call>", True)) - segs.append(("", True)) - if scenario.content is not None: - segs.append((scenario.content, False)) - if scenario.tool_calls: - for tc in scenario.tool_calls: - segs.extend(_gemma4_tool_segments(tc)) - return segs - - -def _build_gemma4(scenario: Scenario, validate: bool = True) -> Sample: - prompt_token_ids = None - if scenario.after_tool_response: - prompt_token_ids = [_GEMMA4_VOCAB["<|tool_response>"]] - sample = _make_sample( - sample_id=f"gemma4-{scenario.id}", - description=scenario.description, - vocab=_GEMMA4_VOCAB, - segments=_gemma4_segments(scenario), - expected_reasoning=scenario.reasoning, - expected_content=_qwen3_expected_content(scenario), - expected_tool_calls=_expected_tc(scenario), - tools=_expected_tools(scenario), - prompt_token_ids=prompt_token_ids, - ) - if validate: - _validate_sample(sample, Gemma4Parser) - return sample - - -def _build_nemotron_v3(scenario: Scenario, validate: bool = True) -> Sample: - return _build_qwen3( - scenario, - name="nemotron_v3", - parser_cls=NemotronV3Parser, - strip_trailing_ws=True, - validate=validate, - ) - - -def _build_granite_thinking(scenario: Scenario, validate: bool = True) -> Sample: - """Granite 4.2: the Nemotron V3 grammar plus leading-newline stripping on - content after ````.""" - sample = _build_qwen3( - scenario, - name="granite_thinking_parser", - parser_cls=GraniteThinkingParser, - strip_trailing_ws=True, - validate=False, - ) - sample = replace( - sample, - expected_content=( - sample.expected_content.lstrip("\n") - if sample.expected_content - else sample.expected_content - ), - content_lstrip="\n", - ) - if validate: - _validate_sample(sample, GraniteThinkingParser) - return sample - - -# ── Seed-OSS (Qwen3 XML grammar with Seed wrapper tokens) ──────────── - -_SEED_OSS_VOCAB: dict[str, int] = { - "": 50, - "": 51, - "": 60, - "": 61, -} - - -def _seed_oss_tool_segments(tc: ToolCallSpec) -> list[tuple[str, bool]]: - parts = [f"\n"] - for key, value in tc.arguments.items(): - parts.append(f"\n{_qwen3_arg_value(value)}") - parts.append("\n\n") - return [ - ("", True), - ("".join(parts), False), - ("", True), - ] - - -def _seed_oss_segments(scenario: Scenario) -> list[tuple[str, bool]]: - segs: list[tuple[str, bool]] = [] - if scenario.reasoning is not None: - segs.append((scenario.reasoning, False)) - if scenario.content is not None or scenario.tool_calls is not None: - segs.append(("", True)) - if scenario.tool_calls is not None and not scenario.tool_calls: - segs.append(("", True)) - segs.append(("", True)) - if scenario.content is not None: - segs.append((scenario.content, False)) - if scenario.tool_calls: - for tc in scenario.tool_calls: - segs.extend(_seed_oss_tool_segments(tc)) - return segs - - -def _build_seed_oss(scenario: Scenario, validate: bool = True) -> Sample: - sample = _make_sample( - sample_id=f"seed_oss-{scenario.id}", - description=scenario.description, - vocab=_SEED_OSS_VOCAB, - segments=_seed_oss_segments(scenario), - expected_reasoning=scenario.reasoning if scenario.reasoning is not None else "", - expected_content=_qwen3_expected_content(scenario), - expected_tool_calls=_expected_tc(scenario), - tools=_expected_tools(scenario), - ) - if validate: - _validate_sample(sample, SeedOssParser) - return sample - - -# ── DeepSeek V4 (DSML tool format) ────────────────────────────────── - -_DSML = "|DSML|" -_DSV4_VOCAB: dict[str, int] = { - "": 128821, - "": 128822, - f"<{_DSML}tool_calls>": 128823, - f"": 128824, -} - - -def _dsv4_param_text(key: str, value: Any, dsml: str = _DSML) -> str: - is_string = isinstance(value, str) - if is_string: - val_str = value - elif isinstance(value, bool): - val_str = "true" if value else "false" - elif isinstance(value, (int, float)): - val_str = str(value) - else: - val_str = json.dumps(value, ensure_ascii=False) - string_attr = "true" if is_string else "false" - return ( - f'<{dsml}parameter name="{key}" string="{string_attr}">' - f"{val_str}\n" - ) - - -def _dsv4_tool_text(tc: ToolCallSpec, dsml: str = _DSML) -> str: - parts = [f'<{dsml}invoke name="{tc.name}">\n'] - for key, value in tc.arguments.items(): - parts.append(_dsv4_param_text(key, value, dsml)) - parts.append(f"\n") - return "".join(parts) - - -def _dsml_tool_segs( - scenario: Scenario, - tag: str, - dsml: str = _DSML, -) -> list[tuple[str, bool]]: - if not scenario.tool_calls: - return [] - parts = ["\n"] - for tc in scenario.tool_calls: - parts.append(_dsv4_tool_text(tc, dsml)) - return [ - (f"<{dsml}{tag}>", True), - ("".join(parts), False), - (f"", True), - ] - - -def _dsv4_segments( - scenario: Scenario, thinking: bool, *, v41: bool = False -) -> list[tuple[str, bool]]: - segs: list[tuple[str, bool]] = [] - - if thinking: - if scenario.reasoning is not None: - segs.append((scenario.reasoning, False)) - if scenario.content is not None or scenario.tool_calls: - segs.append(("", True)) - else: - if scenario.reasoning is not None: - segs.append(("", True)) - segs.append((scenario.reasoning, False)) - segs.append(("", True)) - - if scenario.content is not None: - segs.append((scenario.content, False)) - - segs.extend( - _dsml_tool_segs(scenario, "calls", f"{_DSML} ") - if v41 - else _dsml_tool_segs(scenario, "tool_calls") - ) - return segs - - -def _build_deepseek_v4( - scenario: Scenario, validate: bool = True, *, v41: bool = False -) -> Sample: - thinking = scenario.reasoning is not None - chat_kwargs = {"thinking": thinking} - parser_cls = DeepSeekV41Parser if v41 else DeepSeekV4Parser - model = "deepseek_v41" if v41 else "deepseek_v4" - vocab = ( - { - "": 128821, - "": 128822, - f"<{_DSML} calls>": 128823, - f"": 128824, - } - if v41 - else _DSV4_VOCAB - ) - - if thinking: - expected_reasoning: str | None = scenario.reasoning or "" - else: - expected_reasoning = None - - sample = _make_sample( - sample_id=f"{model}-{scenario.id}", - description=scenario.description, - vocab=vocab, - segments=_dsv4_segments(scenario, thinking, v41=v41), - expected_reasoning=expected_reasoning, - expected_content=_qwen3_expected_content(scenario), - expected_tool_calls=_expected_tc(scenario), - tools=_expected_tools(scenario), - chat_template_kwargs=chat_kwargs, - ) - if validate: - kwargs = {} - if chat_kwargs: - kwargs["chat_template_kwargs"] = chat_kwargs - _validate_sample(sample, parser_cls, **kwargs) - return sample - - -# ── DeepSeek V3.2 (DSML tool format, no reasoning) ────────────────── - -_DSV32_VOCAB: dict[str, int] = { - f"<{_DSML}function_calls>": 128830, - f"": 128831, -} - - -def _dsv32_segments(scenario: Scenario) -> list[tuple[str, bool]]: - segs: list[tuple[str, bool]] = [] - - if scenario.content is not None: - segs.append((scenario.content, False)) - - segs.extend(_dsml_tool_segs(scenario, "function_calls")) - return segs - - -def _build_deepseek_v32(scenario: Scenario, validate: bool = True) -> Sample | None: - if scenario.reasoning is not None: - return None - - sample = _make_sample( - sample_id=f"deepseek_v32-{scenario.id}", - description=scenario.description, - vocab=_DSV32_VOCAB, - segments=_dsv32_segments(scenario), - expected_reasoning=None, - expected_content=_qwen3_expected_content(scenario), - expected_tool_calls=_expected_tc(scenario), - tools=_expected_tools(scenario), - ) - if validate: - _validate_sample(sample, DeepSeekV32Parser) - return sample - - -# ── GLM-4.7 MoE (XML tool format, starts in REASONING) ────────────── - -_GLM47_MOE_VOCAB: dict[str, int] = { - "": 50, - "": 51, - "": 60, - "": 61, - "": 62, - "": 63, - "": 64, - "": 65, -} - - -def _glm47_moe_arg_value(value: Any) -> str: - if isinstance(value, bool): - return "true" if value else "false" - if isinstance(value, (int, float)): - return str(value) - if isinstance(value, str): - return value - return json.dumps(value, ensure_ascii=False) - - -def _glm47_moe_tool_segments(tc: ToolCallSpec) -> list[tuple[str, bool]]: - segs: list[tuple[str, bool]] = [ - ("", True), - (tc.name, False), - ] - for key, value in tc.arguments.items(): - segs.extend( - [ - ("", True), - (key, False), - ("", True), - ("", True), - (_glm47_moe_arg_value(value), False), - ("", True), - ] - ) - segs.append(("", True)) - return segs - - -def _glm47_moe_segments(scenario: Scenario) -> list[tuple[str, bool]]: - segs: list[tuple[str, bool]] = [] - if scenario.reasoning is not None: - segs.append((scenario.reasoning, False)) - if scenario.content is not None or scenario.tool_calls: - segs.append(("", True)) - if scenario.content is not None: - segs.append((scenario.content, False)) - if scenario.tool_calls: - for tc in scenario.tool_calls: - segs.extend(_glm47_moe_tool_segments(tc)) - return segs - - -def _build_glm47_moe(scenario: Scenario, validate: bool = True) -> Sample: - sample = _make_sample( - sample_id=f"glm47_moe-{scenario.id}", - description=scenario.description, - vocab=_GLM47_MOE_VOCAB, - segments=_glm47_moe_segments(scenario), - expected_reasoning=scenario.reasoning if scenario.reasoning is not None else "", - expected_content=_qwen3_expected_content(scenario), - expected_tool_calls=_expected_tc(scenario), - tools=_expected_tools(scenario), - ) - if validate: - _validate_sample(sample, Glm47MoeParser) - return sample - - -# ── Kimi K2 (native tool-call section, starts in REASONING) ────────── - -_KIMI_K2_VOCAB: dict[str, int] = { - "": 50, - "": 51, - "<|tool_calls_section_begin|>": 60, - "<|tool_calls_section_end|>": 61, - "<|tool_call_begin|>": 62, - "<|tool_call_end|>": 63, - "<|tool_call_argument_begin|>": 64, -} - - -def _kimi_k2_tool_segments( - tool_calls: list[ToolCallSpec], -) -> list[tuple[str, bool]]: - segs: list[tuple[str, bool]] = [("<|tool_calls_section_begin|>", True)] - for index, tc in enumerate(tool_calls): - args = json.dumps(tc.arguments, ensure_ascii=False, separators=(",", ":")) - segs.extend( - [ - ("<|tool_call_begin|>", True), - (f"functions.{tc.name}:{index}\n", False), - ("<|tool_call_argument_begin|>", True), - (args, False), - ("<|tool_call_end|>", True), - ] - ) - segs.append(("<|tool_calls_section_end|>", True)) - return segs - - -def _kimi_k2_segments(scenario: Scenario) -> list[tuple[str, bool]]: - segs: list[tuple[str, bool]] = [] - if scenario.reasoning is not None: - segs.append(("", True)) - segs.append((scenario.reasoning, False)) - if scenario.content is not None or scenario.tool_calls is not None: - segs.append(("", True)) - if scenario.content is not None: - segs.append((scenario.content, False)) - if scenario.tool_calls is not None: - segs.extend(_kimi_k2_tool_segments(scenario.tool_calls)) - return segs - - -def _build_kimi_k2( - scenario: Scenario, - validate: bool = True, - thinking: bool = True, -) -> Sample: - expected_reasoning = ( - scenario.reasoning.rstrip() - if (thinking and scenario.reasoning is not None) - else None - ) - if thinking and scenario.reasoning is None: - expected_reasoning = "" - - sample = _make_sample( - sample_id=f"kimi_k2-{scenario.id}", - description=scenario.description, - vocab=_KIMI_K2_VOCAB, - segments=_kimi_k2_segments(scenario), - expected_reasoning=expected_reasoning, - expected_content=_qwen3_expected_content(scenario), - expected_tool_calls=_expected_tc(scenario), - tools=_expected_tools(scenario), - chat_template_kwargs=None if thinking else {"thinking": False}, - ) - if validate: - _validate_sample( - sample, - KimiK2Parser, - chat_template_kwargs=sample.chat_template_kwargs, - ) - return sample - - -_KIMI_K2_SCENARIOS = [ - *SCENARIOS, - Scenario( - id="trailing-reasoning-whitespace", - description="Reasoning trailing whitespace is stripped", - reasoning="Reasoning with trailing whitespace. \n\t", - content="Done.", - ), -] - - -# ── Inkling (typed content blocks, JSON tool payloads) ─────────────────── - -_TML_VOCAB: dict[str, int] = { - "<|message_model|>": 200001, - "<|content_text|>": 200004, - "<|content_model_end_sampling|>": 200006, - "<|content_thinking|>": 200008, - "<|end_message|>": 200010, - "<|content_tool_error|>": 200022, - "<|content_invoke_tool_json|>": 200049, - "<|content_invoke_tool_text|>": 200057, -} - - -def _inkling_block( - segs: list[tuple[str, bool]], - kind_token: str, - body: str, -) -> None: - """Append one Inkling content block; blocks after the first start with - the ``<|message_model|>`` role token (the first block continues the - generation prompt directly).""" - if segs: - segs.append(("<|message_model|>", True)) - segs.append((kind_token, True)) - if body: - segs.append((body, False)) - segs.append(("<|end_message|>", True)) - - -def _inkling_segments(scenario: Scenario) -> list[tuple[str, bool]]: - segs: list[tuple[str, bool]] = [] - if scenario.reasoning is not None: - _inkling_block(segs, "<|content_thinking|>", scenario.reasoning) - if scenario.tool_calls is not None and not scenario.tool_calls: - _inkling_block(segs, "<|content_invoke_tool_json|>", "") - if scenario.content is not None: - _inkling_block(segs, "<|content_text|>", scenario.content) - if scenario.tool_calls: - for tc in scenario.tool_calls: - args = json.dumps(tc.arguments, ensure_ascii=False, separators=(",", ":")) - payload = f'{{"name":"{tc.name}","args":{args}}}' - _inkling_block(segs, "<|content_invoke_tool_json|>", payload) - return segs - - -def _build_inkling(scenario: Scenario, validate: bool = True) -> Sample: - prompt_token_ids = None - if scenario.after_tool_response: - # Prompt ends with a closed tool-response block and the - # generation-prompt role token. - prompt_token_ids = [ - _TML_VOCAB["<|end_message|>"], - _TML_VOCAB["<|message_model|>"], - ] - sample = _make_sample( - sample_id=f"inkling-{scenario.id}", - description=scenario.description, - vocab=_TML_VOCAB, - segments=_inkling_segments(scenario), - expected_reasoning=scenario.reasoning, - expected_content=_qwen3_expected_content(scenario), - expected_tool_calls=_expected_tc(scenario), - tools=_expected_tools(scenario), - prompt_token_ids=prompt_token_ids, - ) - if validate: - _validate_sample(sample, InklingParser) - return sample - - -# ── PLaMo3 (literal nested tool-request wrappers) ─────────────────── - -_PLAMO3_MARKERS = ( - plamo3.BEGIN_THINK, - plamo3.END_THINK, - plamo3.BEGIN_TOOL_REQUESTS, - plamo3.END_TOOL_REQUESTS, - plamo3.BEGIN_TOOL_REQUEST, - plamo3.END_TOOL_REQUEST, - plamo3.BEGIN_TOOL_NAME, - plamo3.END_TOOL_NAME, - plamo3.BEGIN_TOOL_ARGUMENTS, - plamo3.END_TOOL_ARGUMENTS, - plamo3.EOT, -) -_PLAMO3_VOCAB = {marker: 300 + i for i, marker in enumerate(_PLAMO3_MARKERS)} - - -def _plamo3_tool_segments(tc: ToolCallSpec) -> list[tuple[str, bool]]: - args = json.dumps(tc.arguments, ensure_ascii=False, separators=(",", ":")) - return [ - (plamo3.BEGIN_TOOL_REQUEST, True), - (plamo3.BEGIN_TOOL_NAME, True), - (tc.name, False), - (plamo3.END_TOOL_NAME, True), - (plamo3.BEGIN_TOOL_ARGUMENTS, True), - (args, False), - (plamo3.END_TOOL_ARGUMENTS, True), - (plamo3.END_TOOL_REQUEST, True), - ] - - -def _plamo3_segments(scenario: Scenario) -> list[tuple[str, bool]]: - segs: list[tuple[str, bool]] = [ - (plamo3.BEGIN_THINK, True), - (scenario.reasoning or "", False), - (plamo3.END_THINK, True), - ] - if scenario.content is not None: - segs.append((scenario.content, False)) - if scenario.tool_calls is not None: - segs.append((plamo3.BEGIN_TOOL_REQUESTS, True)) - for tool_call in scenario.tool_calls: - segs.extend(_plamo3_tool_segments(tool_call)) - segs.append((plamo3.END_TOOL_REQUESTS, True)) - return segs - - -def _build_plamo3(scenario: Scenario, validate: bool = True) -> Sample: - sample = _make_sample( - sample_id=f"plamo3-{scenario.id}", - description=scenario.description, - vocab=_PLAMO3_VOCAB, - segments=_plamo3_segments(scenario), - expected_reasoning=scenario.reasoning or "", - expected_content=_qwen3_expected_content(scenario), - expected_tool_calls=_expected_tc(scenario), - tools=_expected_tools(scenario), - ) - if validate: - _validate_sample(sample, Plamo3Parser) - return sample - - -# ── Granite (JSON-array tool bodies, no reasoning) ─────────────────────── - -_GRANITE_VOCAB: dict[str, int] = { - "<|tool_call|>": 49154, -} - - -def _granite_segments(scenario: Scenario) -> list[tuple[str, bool]]: - segs: list[tuple[str, bool]] = [] - if scenario.content: - # Granite has no reasoning; prose is plain content preceding the marker. - segs.append((scenario.content, False)) - if scenario.tool_calls: - segs.append(("<|tool_call|>", True)) - payload = json.dumps( - [ - {"name": tc.name, "arguments": tc.arguments} - for tc in scenario.tool_calls - ], - ensure_ascii=False, - separators=(", ", ": "), - ) - segs.append((" " + payload, False)) - return segs - - -def _granite_expected_content(scenario: Scenario) -> str | None: - if scenario.tool_calls: - if not scenario.content: - return None - return scenario.content.strip() or None - return scenario.content - - -def _build_granite(scenario: Scenario, validate: bool = True) -> Sample: - sample = _make_sample( - sample_id=f"granite-{scenario.id}", - description=scenario.description, - vocab=_GRANITE_VOCAB, - segments=_granite_segments(scenario), - expected_reasoning=None, - expected_content=_granite_expected_content(scenario), - expected_tool_calls=_expected_tc(scenario), - tools=_expected_tools(scenario), - ) - if validate: - _validate_sample(sample, GraniteParser) - return sample - - -# Granite has no reasoning, so the shared reasoning-centric SCENARIOS do not -# apply; these exercise the JSON-array tool body (single, parallel, surrounding -# text) instead. -_GRANITE_SCENARIOS: list[Scenario] = [ - Scenario( - id="single-tool", - description="Single tool call", - tool_calls=[_READ_TOOL], - ), - Scenario( - id="parallel-tools", - description="Parallel tool calls in one JSON array", - tool_calls=[_BASH_TOOL, _WEATHER_TOOL], - ), - Scenario( - id="complex-json-args", - description="Tool call with nested objects, arrays, numbers, booleans", - tool_calls=[_COMPLEX_TOOL], - ), - Scenario( - id="content-only", - description="Plain content response without tool calls", - content="Hello! How can I help you today?", - ), - Scenario( - id="surrounding-text", - description="Prose content preceding the tool call", - content="Let me check the weather.", - tool_calls=[_WEATHER_TOOL], - ), -] - - -# ── Registry and public API ────────────────────────────────────────── - -_BUILDERS: dict[str, Any] = { - "deepseek_v32": _build_deepseek_v32, - "deepseek_v4": _build_deepseek_v4, - "deepseek_v41": functools.partial(_build_deepseek_v4, v41=True), - "gemma4": _build_gemma4, - "granite": _build_granite, - "minimax_m2": _build_minimax_m2, - "minimax_m3": _build_minimax_m3, - "nemotron_v3": _build_nemotron_v3, - "granite_thinking_parser": _build_granite_thinking, - "seed_oss": _build_seed_oss, - "glm47_moe": _build_glm47_moe, - "kimi_k2": _build_kimi_k2, - "qwen3": _build_qwen3, - "mimo": _build_mimo, - "step3p5": _build_step3p5, - "inkling": _build_inkling, - "plamo3": _build_plamo3, -} - - -@functools.cache -def build_samples(model: str) -> tuple[Sample, ...]: - """Build all scenario samples for a model, self-validated.""" - builder = _BUILDERS[model] - scenarios = _KIMI_K2_SCENARIOS if model == "kimi_k2" else SCENARIOS - return tuple(s for s in (builder(sc) for sc in scenarios) if s is not None) diff --git a/tests/platforms/test_cpu.py b/tests/platforms/test_cpu.py deleted file mode 100644 index 956436297c..0000000000 --- a/tests/platforms/test_cpu.py +++ /dev/null @@ -1,224 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -import os -from types import SimpleNamespace - -import pytest -import torch - -from vllm.config import CacheConfig, KVTransferConfig, VllmConfig -from vllm.platforms.cpu import CpuPlatform - - -class _FakeModel: - @classmethod - def get_mamba_state_dtype_from_config( - cls, vllm_config: VllmConfig - ) -> tuple[torch.dtype, torch.dtype]: - state_dtype = { - "float16": torch.float16, - "bfloat16": torch.bfloat16, - }.get(vllm_config.cache_config.mamba_ssm_cache_dtype, torch.float32) - return torch.float32, state_dtype - - -class _FakeRegistry: - @staticmethod - def resolve_model_cls( - architecture: str, - model_config: SimpleNamespace, - ) -> tuple[type[_FakeModel], str]: - return _FakeModel, architecture - - -def _cpu_config( - cache_config: CacheConfig, - *, - model_type: str, - resolved_dtype: str, - architecture: str | None = None, - layer_types: tuple[str, ...] = ("linear_attention",), -) -> SimpleNamespace: - model_config = SimpleNamespace( - architecture=architecture or model_type, - has_inner_state=True, - use_mla=False, - hf_config=SimpleNamespace(model_type=model_type), - hf_text_config=SimpleNamespace( - model_type=model_type, - layer_types=layer_types, - ), - registry=_FakeRegistry, - ) - config = VllmConfig(cache_config=cache_config) - config.model_config = model_config - cache_config.mamba_ssm_cache_dtype = resolved_dtype - return config - - -@pytest.mark.parametrize( - ( - "model_type", - "requested_dtype", - "resolved_dtype", - "layer_types", - "expected_dtype", - ), - [ - pytest.param( - "qwen3_5", - "bfloat16", - "bfloat16", - ("linear_attention",), - "bfloat16", - id="gdn-explicit-bf16", - ), - pytest.param( - "qwen3_5", - "float16", - "float16", - ("linear_attention",), - "float16", - id="gdn-explicit-fp16", - ), - pytest.param( - "qwen3_5", - "auto", - "bfloat16", - ("linear_attention",), - "bfloat16", - id="gdn-model-bf16", - ), - pytest.param( - "qwen3_5", - "auto", - "float16", - ("linear_attention",), - "float16", - id="gdn-model-fp16", - ), - pytest.param( - "nemotron_h", - "auto", - "bfloat16", - ("mamba",), - "float32", - id="unsupported-model-bf16", - ), - pytest.param( - "nemotron_h", - "auto", - "float16", - ("mamba",), - "float32", - id="unsupported-model-fp16", - ), - pytest.param( - "nemotron_h", - "bfloat16", - "bfloat16", - ("mamba",), - "float32", - id="unsupported-explicit-bf16", - ), - pytest.param( - "nemotron_h", - "float16", - "float16", - ("mamba",), - "float32", - id="unsupported-explicit-fp16", - ), - ], -) -def test_cpu_accelerated_gdn_dtype_policy( - monkeypatch: pytest.MonkeyPatch, - model_type: str, - requested_dtype: str, - resolved_dtype: str, - layer_types: tuple[str, ...], - expected_dtype: str, -) -> None: - monkeypatch.setattr("torch.cpu._is_avx512_bf16_supported", lambda: True) - cache_config = CacheConfig(mamba_ssm_cache_dtype=requested_dtype) - config = _cpu_config( - cache_config, - model_type=model_type, - resolved_dtype=resolved_dtype, - layer_types=layer_types, - ) - - CpuPlatform.check_and_update_config(config) - assert cache_config.mamba_ssm_cache_dtype == expected_dtype - - -@pytest.mark.parametrize( - ( - "connector", - "extra_config", - "explicit_layout", - "avx512_bf16_supported", - "expected_layout", - ), - [ - pytest.param("NixlConnector", None, None, True, "DS", id="nixl"), - pytest.param("NixlPullConnector", None, None, True, "DS", id="nixl-pull"), - pytest.param("NixlPushConnector", None, None, True, "DS", id="nixl-push"), - pytest.param( - "MultiConnector", - { - "connectors": [ - { - "kv_connector": "ExampleConnector", - "kv_connector_extra_config": {}, - }, - { - "kv_connector": "NixlConnector", - "kv_connector_extra_config": {}, - }, - ] - }, - None, - True, - "DS", - id="multi-nixl", - ), - pytest.param("NixlConnector", None, "SD", True, "SD", id="explicit-override"), - pytest.param("OffloadingConnector", None, None, True, "SD", id="non-nixl"), - pytest.param( - "OffloadingConnector", None, None, False, None, id="non-nixl-no-avx" - ), - ], -) -def test_cpu_conv_state_layout_selection( - monkeypatch: pytest.MonkeyPatch, - connector: str, - extra_config: dict | None, - explicit_layout: str | None, - avx512_bf16_supported: bool, - expected_layout: str | None, -) -> None: - layout_env = "VLLM_SSM_CONV_STATE_LAYOUT" - monkeypatch.setattr( - "torch.cpu._is_avx512_bf16_supported", - lambda: avx512_bf16_supported, - ) - - kv_transfer_config = KVTransferConfig( - kv_connector=connector, - kv_connector_extra_config=extra_config or {}, - kv_role="kv_both", - ) - config = _cpu_config( - CacheConfig(mamba_ssm_cache_dtype="float32"), - model_type="qwen3_5", - resolved_dtype="float32", - ) - config.kv_transfer_config = kv_transfer_config - monkeypatch.delenv(layout_env, raising=False) - if explicit_layout is not None: - monkeypatch.setenv(layout_env, explicit_layout) - - CpuPlatform.check_and_update_config(config) - assert os.environ.get(layout_env) == expected_layout diff --git a/tests/reasoning/test_base_thinking_reasoning_parser.py b/tests/reasoning/test_base_thinking_reasoning_parser.py deleted file mode 100644 index fad9949881..0000000000 --- a/tests/reasoning/test_base_thinking_reasoning_parser.py +++ /dev/null @@ -1,436 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -import pytest -from transformers import AutoTokenizer, PreTrainedTokenizerBase - -from tests.reasoning.utils import run_reasoning_extraction -from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest -from vllm.reasoning.basic_parsers import BaseThinkingReasoningParser - - -# Create a concrete test implementation of BaseThinkingReasoningParser -class TestThinkingReasoningParser(BaseThinkingReasoningParser): - """Test implementation of BaseThinkingReasoningParser.""" - - @property - def start_token(self) -> str: - return "" - - @property - def end_token(self) -> str: - return "" - - -class TestThinkingReasoningParserAlt(BaseThinkingReasoningParser): - """Alternative test implementation with different tokens.""" - - @property - def start_token(self) -> str: - return "" - - @property - def end_token(self) -> str: - return "" - - -# Use a test model -REASONING_MODEL_NAME = "deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B" - - -@pytest.fixture(scope="module") -def test_tokenizer() -> PreTrainedTokenizerBase: - tokenizer: PreTrainedTokenizerBase = AutoTokenizer.from_pretrained( - REASONING_MODEL_NAME - ) - # Add custom test tokens - test_tokens = ["", "", "", ""] - existing_tokens = set(tokenizer.get_vocab().keys()) - new_tokens = [token for token in test_tokens if token not in existing_tokens] - if new_tokens: - tokenizer.add_tokens(new_tokens) - return tokenizer - - -class TestBaseThinkingReasoningParserInit: - """Test initialization and basic properties of - BaseThinkingReasoningParser. - """ - - def test_successful_initialization(self, test_tokenizer): - """Test successful initialization with valid tokens.""" - parser = TestThinkingReasoningParser(test_tokenizer) - assert parser.start_token == "" - assert parser.end_token == "" - assert parser.start_token_id is not None - assert parser.end_token_id is not None - - def test_initialization_with_missing_tokenizer(self): - """Test that initialization fails without tokenizer.""" - with pytest.raises(ValueError, match="model tokenizer must be passed"): - TestThinkingReasoningParser(None) - - def test_initialization_with_missing_tokens(self, test_tokenizer): - """Test that initialization fails when tokens are not in vocabulary.""" - - # Create a parser with tokens not in vocabulary - class MissingTokenParser(BaseThinkingReasoningParser): - @property - def start_token(self) -> str: - return "" - - @property - def end_token(self) -> str: - return "" - - with pytest.raises( - RuntimeError, match="could not locate think start/end tokens" - ): - MissingTokenParser(test_tokenizer) - - def test_initialization_with_empty_tokens(self, test_tokenizer): - """Test that initialization fails with empty token strings.""" - - class EmptyTokenParser(BaseThinkingReasoningParser): - @property - def start_token(self) -> str: - return "" - - @property - def end_token(self) -> str: - return "" - - with pytest.raises( - ValueError, match="start_token and end_token must be defined" - ): - EmptyTokenParser(test_tokenizer) - - -class TestBaseThinkingReasoningParserMethods: - """Test the methods of BaseThinkingReasoningParser.""" - - def test_is_reasoning_end(self, test_tokenizer): - """Test the is_reasoning_end method.""" - parser = TestThinkingReasoningParser(test_tokenizer) - end_token_id = parser.end_token_id - start_token_id = parser.start_token_id - # Test with end token present - assert parser.is_reasoning_end([1, 2, end_token_id, 4]) is True - - # Test without end token - assert parser.is_reasoning_end([1, 2, 3, 4]) is False - - # Test with empty list - assert parser.is_reasoning_end([]) is False - - # Test with interleaved thinking - assert parser.is_reasoning_end([1, start_token_id, 2, end_token_id]) is True - assert parser.is_reasoning_end([1, start_token_id, 2, 3]) is False - assert ( - parser.is_reasoning_end( - [1, start_token_id, 2, end_token_id, 2, 2, start_token_id] - ) - is False - ) - - def test_is_reasoning_end_streaming(self, test_tokenizer): - """Test the is_reasoning_end_streaming method.""" - parser = TestThinkingReasoningParser(test_tokenizer) - end_token_id = parser.end_token_id - start_token_id = parser.start_token_id - - assert ( - parser.is_reasoning_end_streaming([1, 2, end_token_id], [end_token_id]) - is True - ) - assert parser.is_reasoning_end_streaming([1, 2, 3, 4], [4]) is False - assert parser.is_reasoning_end_streaming([], []) is False - assert ( - parser.is_reasoning_end_streaming( - [1, start_token_id, 2, end_token_id], [end_token_id] - ) - is True - ) - assert ( - parser.is_reasoning_end_streaming([1, start_token_id, 2, 3], [3]) is False - ) - assert ( - parser.is_reasoning_end_streaming( - [1, start_token_id, 2, end_token_id, 2, start_token_id, 2], - [2], - ) - is False - ) - assert ( - parser.is_reasoning_end_streaming( - [1, start_token_id, 2, end_token_id, 2, 2], [2] - ) - is False - ) - - def test_count_reasoning_tokens(self, test_tokenizer): - """Count tokens between start/end markers.""" - parser = TestThinkingReasoningParser(test_tokenizer) - start = parser.start_token_id - end = parser.end_token_id - token_ids = [0, start, 11, 12, end, 99] - assert parser.count_reasoning_tokens(token_ids) == 2 - - def test_count_reasoning_tokens_nested(self, test_tokenizer): - """Ensure nested thinking spans count all inner tokens safely.""" - parser = TestThinkingReasoningParser(test_tokenizer) - s = parser.start_token_id - e = parser.end_token_id - token_ids = [s, 1, s, 2, e, 3, e] - # Tokens 1,2,3 are inside reasoning (depth>0) => 3 tokens - assert parser.count_reasoning_tokens(token_ids) == 3 - - def test_extract_content_ids(self, test_tokenizer): - """Test the extract_content_ids method.""" - parser = TestThinkingReasoningParser(test_tokenizer) - end_token_id = parser.end_token_id - - # Test with end token in the middle - input_ids = [1, 2, end_token_id, 4, 5] - content_ids = parser.extract_content_ids(input_ids) - assert content_ids == [4, 5] - - # Test with end token at the end - input_ids = [1, 2, 3, end_token_id] - content_ids = parser.extract_content_ids(input_ids) - assert content_ids == [] - - # Test without end token - input_ids = [1, 2, 3, 4] - content_ids = parser.extract_content_ids(input_ids) - assert content_ids == [] - - # Test with end token as last element (should not extract) - input_ids = [1, 2, 3, end_token_id] - content_ids = parser.extract_content_ids(input_ids) - assert content_ids == [] - - -class TestBaseThinkingReasoningParserExtraction: - """Test reasoning content extraction methods.""" - - def test_extract_reasoning_with_both_tokens(self, test_tokenizer): - """Test extraction when both start and end tokens are present.""" - parser = TestThinkingReasoningParser(test_tokenizer) - request = ChatCompletionRequest(messages=[], model="test-model") - - model_output = "This is reasoningThis is content" - reasoning, content = parser.extract_reasoning(model_output, request) - - assert reasoning == "This is reasoning" - assert content == "This is content" - - def test_extract_reasoning_only_end_token(self, test_tokenizer): - """Test extraction when only end token is present.""" - parser = TestThinkingReasoningParser(test_tokenizer) - request = ChatCompletionRequest(messages=[], model="test-model") - - model_output = "This is reasoningThis is content" - reasoning, content = parser.extract_reasoning(model_output, request) - - assert reasoning == "This is reasoning" - assert content == "This is content" - - def test_extract_reasoning_no_end_token(self, test_tokenizer): - """Test extraction when no end token is present.""" - parser = TestThinkingReasoningParser(test_tokenizer) - request = ChatCompletionRequest(messages=[], model="test-model") - - model_output = "This is just content" - reasoning, content = parser.extract_reasoning(model_output, request) - - assert reasoning == "This is just content" - assert content is None - - def test_extract_reasoning_empty_output(self, test_tokenizer): - """Test extraction with empty output.""" - parser = TestThinkingReasoningParser(test_tokenizer) - request = ChatCompletionRequest(messages=[], model="test-model") - - model_output = "" - reasoning, content = parser.extract_reasoning(model_output, request) - - assert reasoning == "" - assert content is None - - def test_extract_reasoning_only_tokens(self, test_tokenizer): - """Test extraction with only tokens and no content.""" - parser = TestThinkingReasoningParser(test_tokenizer) - request = ChatCompletionRequest(messages=[], model="test-model") - - model_output = "" - reasoning, content = parser.extract_reasoning(model_output, request) - - assert reasoning == "" - assert content is None - - -class TestBaseThinkingReasoningParserStreaming: - """Test streaming functionality of BaseThinkingReasoningParser.""" - - @pytest.mark.parametrize("streaming", [True, False]) - def test_simple_reasoning_extraction(self, test_tokenizer, streaming): - """Test basic reasoning extraction in both - streaming and non-streaming modes. - """ - parser = TestThinkingReasoningParser(test_tokenizer) - - model_output = [ - "", - "Some ", - "reasoning ", - "content", - "", - "Final ", - "answer", - ] - - reasoning, content = run_reasoning_extraction( - parser, model_output, streaming=streaming - ) - - assert reasoning == "Some reasoning content" - assert content == "Final answer" - - def test_streaming_with_incremental_deltas(self, test_tokenizer): - """Test streaming processing with small incremental deltas.""" - parser = TestThinkingReasoningParser(test_tokenizer) - - deltas = [ - "", - "Some ", - "reasoning ", - "content", - "", - "Final ", - "answer", - ] - - reasoning, content = run_reasoning_extraction(parser, deltas, streaming=True) - - assert reasoning == "Some reasoning content" - assert content == "Final answer" - - def test_streaming_with_start_token(self, test_tokenizer): - """Test streaming with start token included.""" - parser = TestThinkingReasoningParser(test_tokenizer) - - deltas = [ - "", - "Some ", - "reasoning", - "", - "Answer", - ] - - reasoning, content = run_reasoning_extraction(parser, deltas, streaming=True) - - assert reasoning == "Some reasoning" - assert content == "Answer" - - def test_streaming_no_end_token(self, test_tokenizer): - """Test streaming when no end token is encountered.""" - parser = TestThinkingReasoningParser(test_tokenizer) - - deltas = [ - "", - "Some ", - "reasoning ", - "without ", - "end", - ] - - reasoning, content = run_reasoning_extraction(parser, deltas, streaming=True) - - assert reasoning == "Some reasoning without end" - assert content is None - - def test_streaming_only_end_token(self, test_tokenizer): - """Test streaming when only end token appears.""" - parser = TestThinkingReasoningParser(test_tokenizer) - - deltas = [ - "", - "Reasoning ", - "content", - "", - "Final", - ] - - reasoning, content = run_reasoning_extraction(parser, deltas, streaming=True) - - assert reasoning == "Reasoning content" - assert content == "Final" - - -class TestBaseThinkingReasoningParserMultipleImplementations: - """Test that multiple implementations of - BaseThinkingReasoningParser work correctly. - """ - - def test_different_token_implementations(self, test_tokenizer): - """Test that different implementations - with different tokens work independently. - """ - parser1 = TestThinkingReasoningParser(test_tokenizer) - parser2 = TestThinkingReasoningParserAlt(test_tokenizer) - - # Test parser1 - model_output1 = "Reasoning1Content1" - reasoning1, content1 = run_reasoning_extraction(parser1, [model_output1]) - assert reasoning1 == "Reasoning1" - assert content1 == "Content1" - - # Test parser2 - model_output2 = "Reasoning2Content2" - reasoning2, content2 = run_reasoning_extraction(parser2, [model_output2]) - assert reasoning2 == "Reasoning2" - assert content2 == "Content2" - - # Verify tokens are different - assert parser1.start_token != parser2.start_token - assert parser1.end_token != parser2.end_token - assert parser1.start_token_id != parser2.start_token_id - assert parser1.end_token_id != parser2.end_token_id - - -class TestBaseThinkingReasoningParserEdgeCases: - """Test edge cases and error conditions.""" - - def test_multiple_end_tokens(self, test_tokenizer): - """Test behavior with multiple end tokens.""" - parser = TestThinkingReasoningParser(test_tokenizer) - - model_output = "FirstMiddleLast" - reasoning, content = run_reasoning_extraction(parser, [model_output]) - - # Should stop at first end token - assert reasoning == "First" - assert content == "MiddleLast" - - def test_nested_tokens(self, test_tokenizer): - """Test behavior with nested-like token patterns.""" - parser = TestThinkingReasoningParser(test_tokenizer) - - model_output = "OuterInnerContent" - reasoning, content = run_reasoning_extraction(parser, [model_output]) - - # Should process normally, start from first start token - assert reasoning == "OuterInner" - assert content == "Content" - - def test_malformed_tokens(self, test_tokenizer): - """Test behavior with malformed token-like strings.""" - parser = TestThinkingReasoningParser(test_tokenizer) - - model_output = "Not a real tokenContent" - reasoning, content = run_reasoning_extraction(parser, [model_output]) - - # Should treat as regular content since tokens don't match exactly - assert reasoning == ("Not a real tokenContent") - assert content is None diff --git a/tests/test_pcp_dp.py b/tests/test_pcp_dp.py deleted file mode 100644 index f6eed57a6f..0000000000 --- a/tests/test_pcp_dp.py +++ /dev/null @@ -1,161 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from types import SimpleNamespace - -import pytest -import torch - -from vllm.config import ParallelConfig -from vllm.distributed.device_communicators.all2all import AgRsAll2AllManager -from vllm.forward_context import DPMetadata -from vllm.v1.attention.ops.pcp import maybe_gather_mla_latent_cache_inputs - - -@pytest.mark.parametrize( - "pcp_size,sp_size,enable_ep,expected", - [ - (1, 1, True, [5, 7]), - (1, 2, True, [3, 3, 4, 4]), - (2, 1, False, [10, 14]), - (2, 1, True, [5, 5, 7, 7]), - (2, 2, True, [3, 3, 3, 3, 4, 4, 4, 4]), - ], -) -def test_dispatch_sizes_expand_pcp_before_tp(pcp_size, sp_size, enable_ep, expected): - config = ParallelConfig( - distributed_executor_backend="mp", - data_parallel_size=2, - prefill_context_parallel_size=pcp_size, - tensor_parallel_size=sp_size, - enable_expert_parallel=enable_ep, - ) - metadata = DPMetadata.make(config, 5, torch.tensor([5, 7])) - with metadata.sp_local_sizes(sp_size, pcp_size=pcp_size, use_ep=enable_ep) as sizes: - assert sizes == expected - assert metadata.local_sizes is None - assert metadata.num_tokens_across_dp_cpu.tolist() == [5, 7] - - -@pytest.mark.parametrize( - "dp_size,pcp_size,tp_size,use_ep,is_sp,expected", - [ - (2, 2, 1, True, False, "ep"), - (2, 2, 2, True, True, "ep"), - (2, 2, 2, False, False, "dp"), - (2, 1, 2, True, False, "dp"), - (1, 2, 2, True, False, "pcp"), - (2, 2, 2, True, False, None), - ], -) -def test_dispatch_reuses_existing_groups( - dp_size, pcp_size, tp_size, use_ep, is_sp, expected, monkeypatch -): - groups = { - "dp": SimpleNamespace(world_size=dp_size), - "pcp": SimpleNamespace(world_size=pcp_size), - "ep": object(), - } - for name, group in groups.items(): - monkeypatch.setattr( - f"vllm.distributed.device_communicators.all2all.get_{name}_group", - lambda group=group: group, - ) - manager = AgRsAll2AllManager.__new__(AgRsAll2AllManager) - manager.dp_world_size = dp_size - manager.tp_group = SimpleNamespace(world_size=tp_size) - manager.use_ep = use_ep - if expected is None: - with pytest.raises(AssertionError, match="requires sequence-parallel MoE"): - manager._get_comm_group(is_sp) - else: - assert manager._get_comm_group(is_sp) is groups[expected] - - -@pytest.mark.parametrize("enable_ep", [False, True]) -def test_ag_rs_dispatch_and_combine_use_dp_pcp_sizes(monkeypatch, enable_ep): - calls = [] - local_tokens = [20, 21] if enable_ep else [20, 21, 22, 23] - - class FakeGroup: - world_size = 4 if enable_ep else 2 - rank_in_group = 2 if enable_ep else 1 - - def all_gatherv(self, tensors, dim, sizes): - calls.append(("gather", sizes)) - return [torch.tensor([10, 11, 20, 21, 22, 23]) for _ in tensors] - - def reduce_scatterv(self, tensor, dim, sizes): - calls.append(("scatter", sizes)) - return tensor[2 : 2 + len(local_tokens)] - - config = ParallelConfig( - distributed_executor_backend="mp", - data_parallel_size=2, - data_parallel_rank=1, - prefill_context_parallel_size=2, - enable_expert_parallel=enable_ep, - ) - metadata = DPMetadata.make(config, 2, torch.tensor([1, 2])) - manager = AgRsAll2AllManager.__new__(AgRsAll2AllManager) - manager.dp_world_size = 2 - manager.use_ep = enable_ep - - manager.tp_group = SimpleNamespace(world_size=1) - for name in ("dp", "ep"): - monkeypatch.setattr( - f"vllm.distributed.device_communicators.all2all.get_{name}_group", - FakeGroup, - ) - monkeypatch.setattr( - "vllm.distributed.device_communicators.all2all.get_pcp_group", - lambda: SimpleNamespace(world_size=2), - ) - monkeypatch.setattr( - "vllm.distributed.device_communicators.all2all.get_forward_context", - lambda: SimpleNamespace(dp_metadata=metadata), - ) - with metadata.sp_local_sizes(1, pcp_size=2, use_ep=enable_ep): - hidden_states, _, _ = manager.dispatch( - torch.tensor(local_tokens), - torch.ones(len(local_tokens)), - torch.zeros(len(local_tokens)), - ) - combined = manager.combine(hidden_states) - assert combined.tolist() == local_tokens - sizes = [1, 1, 2, 2] if enable_ep else [2, 4] - assert calls == [("gather", sizes), ("scatter", sizes)] - - -@pytest.mark.parametrize("slots", [[3, 4], [3, 4, -1, -1, -1, -1]]) -def test_decode_cache_write_ignores_dp_padding(slots): - kv = torch.arange(12).reshape(6, 2) - pe = torch.arange(6).reshape(6, 1, 1) - slots = torch.tensor(slots) - cache_kv, cache_pe, cache_slots = maybe_gather_mla_latent_cache_inputs( - kv, pe, slots, num_decode_tokens=2, use_pcp=True - ) - torch.testing.assert_close(cache_kv, kv[:2]) - torch.testing.assert_close(cache_pe, pe[:2]) - torch.testing.assert_close(cache_slots, slots[:2]) - - -def test_expanded_slot_mapping_keeps_pcp_prefill_padding(monkeypatch): - calls = [] - - def all_gather(tensor, dim): - calls.append(tensor.shape[0]) - return torch.cat((tensor, tensor), dim=dim) - - monkeypatch.setattr( - "vllm.v1.attention.ops.pcp.get_pcp_group", - lambda: SimpleNamespace(world_size=2, all_gather=all_gather), - ) - kv = torch.zeros(3, 2) # Two decodes followed by one PCP padding row. - pe = torch.zeros(3, 1, 1) - slots = torch.tensor([3, 4, 8, 3, 4, -1]) - cache_kv, _, cache_slots = maybe_gather_mla_latent_cache_inputs( - kv, pe, slots, num_decode_tokens=2, use_pcp=True - ) - assert calls == [1, 1] - assert cache_kv.shape == (4, 2) - assert cache_slots.tolist() == [3, 4, 8, -1] diff --git a/tests/transformers_utils/processors/test_pixtral.py b/tests/transformers_utils/processors/test_pixtral.py deleted file mode 100644 index 333308868e..0000000000 --- a/tests/transformers_utils/processors/test_pixtral.py +++ /dev/null @@ -1,65 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -import pytest -import transformers.image_utils -from PIL import Image - -from vllm.transformers_utils.processors.pixtral import MistralCommonImageProcessor - - -@pytest.fixture(scope="module") -def image_processor() -> MistralCommonImageProcessor: - return MistralCommonImageProcessor(mm_encoder=None) - - -def test_fetch_images_passes_through_decoded_image( - image_processor: MistralCommonImageProcessor, -): - image = Image.new("RGB", (4, 4)) - result = image_processor.fetch_images(image) - assert result is image - - -def test_fetch_images_recurses_over_list( - image_processor: MistralCommonImageProcessor, -): - a = Image.new("RGB", (4, 4)) - b = Image.new("RGB", (8, 8)) - result = image_processor.fetch_images([a, b]) - assert isinstance(result, list) - assert len(result) == 2 - assert result[0] is a - assert result[1] is b - - -def test_fetch_images_recurses_over_nested_list( - image_processor: MistralCommonImageProcessor, -): - a = Image.new("RGB", (4, 4)) - b = Image.new("RGB", (8, 8)) - result = image_processor.fetch_images([[a], [b]]) - assert result == [[a], [b]] - - -def test_fetch_images_str_delegates_to_load_image( - monkeypatch, image_processor: MistralCommonImageProcessor -): - sentinel = Image.new("RGB", (2, 2)) - received: dict[str, object] = {} - - def fake_load_image(path): - received["path"] = path - return sentinel - - monkeypatch.setattr(transformers.image_utils, "load_image", fake_load_image) - - result = image_processor.fetch_images("/tmp/fake.png") - assert result is sentinel - assert received["path"] == "/tmp/fake.png" - - -def test_fetch_images_rejects_unsupported_type( - image_processor: MistralCommonImageProcessor, -): - with pytest.raises(TypeError, match="only a single or a list"): - image_processor.fetch_images(42) diff --git a/tests/transformers_utils/processors/test_voxtral.py b/tests/transformers_utils/processors/test_voxtral.py deleted file mode 100644 index 015d3b005e..0000000000 --- a/tests/transformers_utils/processors/test_voxtral.py +++ /dev/null @@ -1,127 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Unit tests for ``MistralCommonFeatureExtractor.fetch_audio``. - -``transformers>=5.10`` adds a ``ProcessorMixin.prepare_inputs_layout`` helper -that calls ``self.feature_extractor.fetch_audio(...)`` unconditionally. The -duck-typed :class:`MistralCommonFeatureExtractor` previously did not implement -that method, so loading any voxtral model under transformers 5.10.x raised -``AttributeError: 'MistralCommonFeatureExtractor' object has no attribute -'fetch_audio'``. These tests pin the new ``fetch_audio`` method to the same -contract as ``transformers.SequenceFeatureExtractor.fetch_audio``. -""" - -import numpy as np -import pytest -import torch - -from vllm.tokenizers.mistral import MistralTokenizer -from vllm.transformers_utils.processors.voxtral import ( - MistralCommonFeatureExtractor, -) - - -@pytest.fixture(scope="module") -def feature_extractor() -> MistralCommonFeatureExtractor: - tokenizer = MistralTokenizer.from_pretrained("mistralai/Voxtral-Mini-3B-2507") - return MistralCommonFeatureExtractor(tokenizer.instruct.audio_encoder) - - -@pytest.mark.parametrize( - "audio", - [ - np.zeros(1024, dtype=np.float32), - torch.zeros(1024), - [0.0, 1.0, 2.0], - ], - ids=["numpy_array", "torch_tensor", "list_of_floats"], -) -def test_fetch_audio_passes_through( - feature_extractor: MistralCommonFeatureExtractor, audio -): - result = feature_extractor.fetch_audio(audio) - assert result is audio - - -def test_fetch_audio_recurses_over_list_of_arrays( - feature_extractor: MistralCommonFeatureExtractor, -): - a = np.zeros(8, dtype=np.float32) - b = np.ones(8, dtype=np.float32) - result = feature_extractor.fetch_audio([a, b]) - assert isinstance(result, list) - assert len(result) == 2 - assert result[0] is a - assert result[1] is b - - -def test_fetch_audio_uses_self_sampling_rate_when_none( - monkeypatch, feature_extractor: MistralCommonFeatureExtractor -): - """If ``sampling_rate`` is None, ``self.sampling_rate`` must be used. - - Verified indirectly via the recursion path: when we pass a list of arrays - without sampling_rate, recursive calls receive the resolved rate. - """ - captured: list[int | None] = [] - original = feature_extractor.fetch_audio - - def spy(audio, sampling_rate=None): - captured.append(sampling_rate) - return original(audio, sampling_rate=sampling_rate) - - monkeypatch.setattr(feature_extractor, "fetch_audio", spy) - feature_extractor.fetch_audio([np.zeros(4, dtype=np.float32)]) - # Top-level call has sampling_rate=None; inner recursive call sees the - # resolved rate from self.sampling_rate. - assert captured[0] is None - assert captured[1] == 16000 - - -def test_fetch_audio_explicit_sampling_rate_propagates( - monkeypatch, feature_extractor: MistralCommonFeatureExtractor -): - captured: list[int | None] = [] - original = feature_extractor.fetch_audio - - def spy(audio, sampling_rate=None): - captured.append(sampling_rate) - return original(audio, sampling_rate=sampling_rate) - - monkeypatch.setattr(feature_extractor, "fetch_audio", spy) - feature_extractor.fetch_audio([np.zeros(4, dtype=np.float32)], sampling_rate=8000) - assert captured[0] == 8000 - assert captured[1] == 8000 - - -def test_fetch_audio_rejects_unsupported_type( - feature_extractor: MistralCommonFeatureExtractor, -): - with pytest.raises(TypeError, match="only a numpy array"): - feature_extractor.fetch_audio(42) - - -def test_fetch_audio_str_delegates_to_load_audio( - monkeypatch, feature_extractor: MistralCommonFeatureExtractor -): - """A str input must round-trip through ``transformers.audio_utils.load_audio``. - - We monkey-patch ``load_audio`` so the test stays offline (no real URL/path - fetched) and still asserts the delegation contract. - """ - sentinel = np.array([0.5, -0.5], dtype=np.float32) - received: dict[str, object] = {} - - def fake_load_audio(path, sampling_rate=None): - received["path"] = path - received["sampling_rate"] = sampling_rate - return sentinel - - import transformers.audio_utils - - monkeypatch.setattr(transformers.audio_utils, "load_audio", fake_load_audio) - - result = feature_extractor.fetch_audio("/tmp/fake.wav") - assert result is sentinel - assert received["path"] == "/tmp/fake.wav" - assert received["sampling_rate"] == 16000 diff --git a/tests/transformers_utils/test_bailing_moe_v3_vl_config.py b/tests/transformers_utils/test_bailing_moe_v3_vl_config.py deleted file mode 100644 index f10fd408ed..0000000000 --- a/tests/transformers_utils/test_bailing_moe_v3_vl_config.py +++ /dev/null @@ -1,159 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -import copy -import json -from pathlib import Path -from typing import Any - -import pytest -from transformers import AutoConfig - -from vllm.transformers_utils.config import get_config, uses_mrope -from vllm.transformers_utils.configs.bailing_moe_v3_vl import ( - BailingMoeV3TextConfig, - BailingMoeV3VisionConfig, - BailingMoeV3VLConfig, -) - - -def _bailing_v3_vl_config() -> dict[str, Any]: - return { - "architectures": ["BailingMoeV3VLForConditionalGeneration"], - "auto_map": { - "AutoConfig": ("configuration_bailing_moe_v3_vl.BailingMoeV3VLConfig") - }, - "model_type": "bailing_moe_v3_vl", - "mrope_section": [8, 12, 12], - "text_config": { - "num_hidden_layers": 42, - "layer_group_size": 6, - "rope_theta": 6_000_000, - }, - "vision_config": {"disable_merger_proj": True}, - } - - -def _write_config(path: Path, config: dict[str, Any]) -> None: - path.mkdir() - (path / "config.json").write_text(json.dumps(config), encoding="utf-8") - - -def test_bailing_v3_vl_config_loads_without_remote_code(tmp_path: Path): - model_path = tmp_path / "model" - _write_config(model_path, _bailing_v3_vl_config()) - - config = get_config(model_path, trust_remote_code=False) - - assert isinstance(config, BailingMoeV3VLConfig) - assert isinstance(config.text_config, BailingMoeV3TextConfig) - assert isinstance(config.vision_config, BailingMoeV3VisionConfig) - assert isinstance( - AutoConfig.from_pretrained(model_path, trust_remote_code=False), - BailingMoeV3VLConfig, - ) - assert config.mrope_section == [8, 12, 12] - assert uses_mrope(config) - - text_config = config.text_config - assert text_config.model_type == "bailing_hybrid" - assert text_config.architectures == ["BailingMoeV3ForCausalLM"] - assert text_config.rope_parameters["mrope_section"] == [8, 12, 12] - assert text_config.rope_parameters["rope_theta"] == 6_000_000 - assert text_config.layer_types.count("linear_attention") == 35 - assert [ - layer_idx - for layer_idx, layer_type in enumerate(text_config.layer_types) - if layer_type == "full_attention" - ] == [5, 11, 17, 23, 29, 35, 41] - assert config.vision_config.disable_merger_proj is True - - normalized = config.to_dict() - restored = BailingMoeV3VLConfig.from_dict(normalized) - assert isinstance(restored.text_config, BailingMoeV3TextConfig) - assert isinstance(restored.vision_config, BailingMoeV3VisionConfig) - assert restored.to_dict() == normalized - - -def test_bailing_v3_vl_config_preserves_explicit_nested_values(): - config_dict = _bailing_v3_vl_config() - config_dict["architectures"] = ["CustomBailingVLForConditionalGeneration"] - text_config = config_dict["text_config"] - text_config["architectures"] = ["CustomBailingForCausalLM"] - text_config["layer_types"] = (["linear_attention"] * 5 + ["full_attention"]) * 7 - text_config["rope_theta"] = 1_000_000 - text_config["rope_parameters"] = { - "rope_type": "default", - "rope_theta": 1_000_000, - "mrope_section": [8, 12, 12], - } - original = copy.deepcopy(config_dict) - - config = BailingMoeV3VLConfig(**config_dict) - - assert config_dict == original - assert config.architectures == ["CustomBailingVLForConditionalGeneration"] - assert config.text_config.architectures == ["CustomBailingForCausalLM"] - assert config.text_config.layer_types == text_config["layer_types"] - assert config.text_config.rope_parameters["rope_theta"] == 1_000_000 - assert config.text_config.rope_parameters["mrope_section"] == [8, 12, 12] - - -def test_bailing_v3_vl_config_normalizes_config_instance(): - text_config = BailingMoeV3TextConfig() - - config = BailingMoeV3VLConfig( - text_config=text_config, - mrope_section=[8, 12, 12], - ) - - assert config.text_config is text_config - assert text_config.rope_parameters["mrope_section"] == [8, 12, 12] - - -def test_bailing_v3_text_config_trailing_and_legacy_layers(): - expected = ["linear_attention", "linear_attention", "full_attention"] * 2 + [ - "full_attention" - ] * 2 - text_config = BailingMoeV3TextConfig( - num_hidden_layers=8, - layer_group_size=3, - ) - assert text_config.layer_types == expected - - text_config = BailingMoeV3TextConfig( - num_hidden_layers=8, - layer_group_size=3, - layers_block_type=["mamba", "mamba", "attention"] * 2 + ["attention"] * 2, - ) - assert text_config.layer_types == expected - - -def test_bailing_v3_text_config_accepts_legacy_rope_type(): - text_config = BailingMoeV3TextConfig(rope_scaling={"type": "linear", "factor": 2.0}) - - assert text_config.rope_parameters["type"] == "linear" - assert text_config.rope_parameters["rope_type"] == "linear" - - -def test_bailing_v3_vl_config_rejects_invalid_values(): - with pytest.raises(ValueError, match="layer_group_size must be positive"): - BailingMoeV3TextConfig(layer_group_size=0) - - with pytest.raises(ValueError, match="must match the layer_group_size schedule"): - BailingMoeV3TextConfig( - num_hidden_layers=8, - layer_group_size=3, - layer_types=["full_attention"] * 8, - ) - - with pytest.raises(ValueError, match="conflicting M-RoPE sections"): - BailingMoeV3VLConfig( - text_config={ - "rope_parameters": { - "rope_type": "default", - "mrope_section": [4, 14, 14], - } - }, - mrope_section=[8, 12, 12], - ) diff --git a/tests/transformers_utils/test_dspark_mla_config.py b/tests/transformers_utils/test_dspark_mla_config.py deleted file mode 100644 index 9e44acb55f..0000000000 --- a/tests/transformers_utils/test_dspark_mla_config.py +++ /dev/null @@ -1,153 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -import json - -import pytest - -from vllm.config import ModelConfig, ParallelConfig, SpeculativeConfig -from vllm.transformers_utils.config import get_config -from vllm.transformers_utils.configs.k3_dspark import K3DSparkConfig - - -def _write_dspark_config(path, **overrides): - path.mkdir() - config = { - "architectures": ["K3DSparkModel"], - "model_type": "k3_dspark", - "hidden_size": 7168, - "intermediate_size": 14336, - "num_hidden_layers": 5, - "num_attention_heads": 96, - "num_key_value_heads": 96, - "q_lora_rank": 1536, - "kv_lora_rank": 512, - "qk_nope_head_dim": 128, - "qk_rope_head_dim": 64, - "v_head_dim": 128, - "vocab_size": 163840, - "rms_norm_eps": 1e-5, - "max_position_embeddings": 32768, - "rope_theta": 50000.0, - "num_target_layers": 5, - "target_hidden_size": 7168, - "target_num_hidden_layers": 93, - "target_layer_ids": [2, 23, 47, 71, 89], - "markov_rank": 256, - "draft_vocab_size": 163840, - "torch_dtype": "bfloat16", - } - config.update(overrides) - (path / "config.json").write_text(json.dumps(config)) - - -def _write_target_config(path): - path.mkdir() - config = { - "architectures": ["LlamaForCausalLM"], - "model_type": "llama", - "hidden_size": 7168, - "intermediate_size": 14336, - "num_hidden_layers": 93, - "num_attention_heads": 56, - "num_key_value_heads": 8, - "vocab_size": 163840, - "max_position_embeddings": 32768, - "torch_dtype": "bfloat16", - } - (path / "config.json").write_text(json.dumps(config)) - - -def test_dspark_mla_config_loads_from_local_json(tmp_path): - draft_path = tmp_path / "draft" - _write_dspark_config(draft_path) - - config = get_config(draft_path, trust_remote_code=False) - - assert isinstance(config, K3DSparkConfig) - assert config.model_type == "k3_dspark" - assert config.architectures == ["K3DSparkModel"] - assert config.hidden_act == "silu" - assert config.rope_parameters == { - "rope_type": "default", - "rope_theta": 50000.0, - } - assert config.n_routed_experts == 0 - assert config.draft_vocab_size == config.vocab_size - - -@pytest.mark.parametrize( - "overrides", - [ - {"mla_use_nope": True}, - {"mla_use_output_gate": True}, - {"mla_use_qk_norm": True}, - {"dspark_bonus_anchor": True}, - {"q_lora_rank": None}, - {"draft_vocab_size": 8192}, - {"target_layer_ids": []}, - {"num_target_layers": 4}, - ], -) -def test_dspark_mla_rejects_unsupported_checkpoint_options(tmp_path, overrides): - draft_path = tmp_path / "draft" - _write_dspark_config(draft_path, **overrides) - - with pytest.raises(ValueError, match="MLA DSpark"): - get_config(draft_path, trust_remote_code=False) - - -@pytest.mark.parametrize( - ("num_attention_heads", "expected_local_heads"), - [(64, 8), (96, 12)], - ids=["released-64-head", "block5-96-head"], -) -def test_dspark_mla_uses_latent_kv_geometry( - tmp_path, num_attention_heads, expected_local_heads -): - draft_path = tmp_path / "draft" - _write_dspark_config( - draft_path, - num_attention_heads=num_attention_heads, - num_key_value_heads=num_attention_heads, - ) - model_config = ModelConfig( - model=str(draft_path), - tokenizer_mode="skip", - runner="draft", - max_model_len=32768, - ) - - assert model_config.is_deepseek_mla - assert model_config.use_mla - assert model_config.get_head_size() == 576 - # external_launcher skips ParallelConfig's local-GPU-count check so the - # config logic can be exercised at TP8 on a single-GPU test node. - parallel_config = ParallelConfig( - tensor_parallel_size=8, distributed_executor_backend="external_launcher" - ) - assert model_config.get_num_kv_heads(parallel_config) == 1 - assert model_config.get_num_attention_heads(parallel_config) == expected_local_heads - assert model_config.get_num_experts() == 0 - - -def test_dspark_mla_speculative_config_preserves_architecture(tmp_path): - target_path = tmp_path / "target" - draft_path = tmp_path / "draft" - _write_target_config(target_path) - _write_dspark_config(draft_path) - target_config = ModelConfig( - model=str(target_path), tokenizer_mode="skip", max_model_len=32768 - ) - speculative_config = SpeculativeConfig( - model=str(draft_path), - method="dspark", - num_speculative_tokens=8, - target_model_config=target_config, - target_parallel_config=ParallelConfig(), - ) - - assert speculative_config.parallel_drafting - assert speculative_config.draft_model_config.architectures == ["K3DSparkModel"] - assert speculative_config.draft_model_config.hf_config.model_type == "k3_dspark" - assert speculative_config.draft_model_config.use_mla diff --git a/tests/v1/core/test_prefix_caching.py b/tests/v1/core/test_prefix_caching.py index 52455c6083..2add808506 100644 --- a/tests/v1/core/test_prefix_caching.py +++ b/tests/v1/core/test_prefix_caching.py @@ -6,8 +6,7 @@ import copy from collections.abc import Callable from dataclasses import replace from math import lcm -from types import MethodType, SimpleNamespace -from unittest.mock import MagicMock +from types import SimpleNamespace import pytest import torch @@ -15,21 +14,11 @@ import torch import vllm.v1.core.kv_cache_manager as kv_cache_manager import vllm.v1.core.kv_cache_utils as kv_cache_utils from vllm.distributed.kv_events import ( - MEDIUM_CPU, MEDIUM_GPU, AllBlocksCleared, BlockRemoved, BlockStored, ) -from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.connector import ( - HiSparseConnector, - HiSparseConnectorScheduler, - HiSparseConnectorWorkerMetadata, -) -from vllm.distributed.kv_transfer.kv_connector.v1.multi_connector import MultiConnector -from vllm.distributed.kv_transfer.kv_connector.v1.offloading_connector import ( - OffloadingConnector, -) from vllm.lora.request import LoRARequest from vllm.multimodal.inputs import ( MultiModalFeatureSpec, @@ -51,23 +40,17 @@ from vllm.v1.core.kv_cache_utils import ( init_none_hash, make_block_hash_with_group_id, ) -from vllm.v1.core.sched.output import SchedulerOutput from vllm.v1.core.sched.scheduler import Scheduler from vllm.v1.hisparse.coordinator import get_hisparse_coordinator from vllm.v1.kv_cache_interface import ( FullAttentionSpec, - HiSparseHotSpec, - HiSparseResidentSpec, KVCacheConfig, - KVCacheGroupRole, KVCacheGroupSpec, KVCacheSpecKind, MambaSpec, MLAAttentionSpec, SlidingWindowSpec, ) -from vllm.v1.outputs import KVConnectorOutput -from vllm.v1.request import RequestStatus pytestmark = pytest.mark.cpu_test @@ -136,10 +119,7 @@ def make_kv_cache_manager(kv_cache_config: KVCacheConfig, **kwargs) -> KVCacheMa kv_cache_config, prefix_cache_retention_interval=kwargs.pop("retention_interval"), ) - manager = KVCacheManager(kv_cache_config, **kwargs) - if kv_cache_config.hisparse_host_num_blocks is not None: - get_hisparse_coordinator(manager) - return manager + return KVCacheManager(kv_cache_config, **kwargs) def make_kv_cache_config(block_size: int, num_blocks: int) -> KVCacheConfig: @@ -163,90 +143,6 @@ def make_kv_cache_config(block_size: int, num_blocks: int) -> KVCacheConfig: HISPARSE_BLOCK_SIZE = 16 -def make_hisparse_kv_cache_config( - num_blocks: int, - host_num_blocks: int, - *, - transfer_device_cache: bool = False, - num_resident_groups: int = 1, -) -> KVCacheConfig: - source_spec = FullAttentionSpec( - block_size=HISPARSE_BLOCK_SIZE, - num_kv_heads=1, - head_size=1, - dtype=torch.float32, - ) - groups = [ - KVCacheGroupSpec( - ["source"], - source_spec, - role=KVCacheGroupRole.HISPARSE_SOURCE, - host_resident=True, - ), - KVCacheGroupSpec( - ["indexer"], - source_spec, - enable_kv_transfer=transfer_device_cache, - role=KVCacheGroupRole.HISPARSE_INDEXER, - ), - ] - groups.append( - KVCacheGroupSpec( - ["resident"], - HiSparseResidentSpec( - block_size=HISPARSE_BLOCK_SIZE, - page_size=HISPARSE_BLOCK_SIZE * 4, - ), - enable_kv_transfer=transfer_device_cache, - ) - ) - groups.append( - KVCacheGroupSpec( - ["hot"], - HiSparseHotSpec( - block_size=HISPARSE_BLOCK_SIZE, - page_size=HISPARSE_BLOCK_SIZE * 4, - blocks_per_request=2, - ), - enable_kv_transfer=False, - ) - ) - for group_idx in range(1, num_resident_groups): - groups.extend( - replace( - group, layer_names=[f"{name}_{group_idx}" for name in group.layer_names] - ) - for group in groups[2:4] - ) - return KVCacheConfig( - num_blocks=num_blocks, - hisparse_host_num_blocks=host_num_blocks, - kv_cache_tensors=[], - kv_cache_groups=groups, - ) - - -def make_hisparse_kv_cache_manager( - num_blocks: int, - host_num_blocks: int, - *, - max_model_len: int = 128, - max_in_flight_tokens: int | None = None, - enable_caching: bool = False, - num_prefill_lookahead: int = 0, - **config_kwargs, -) -> KVCacheManager: - config = make_hisparse_kv_cache_config(num_blocks, host_num_blocks, **config_kwargs) - return make_kv_cache_manager( - config, - max_model_len=max_model_len, - max_in_flight_tokens=max_in_flight_tokens, - enable_caching=enable_caching, - hash_block_size=HISPARSE_BLOCK_SIZE, - num_prefill_lookahead=num_prefill_lookahead, - ) - - def _allocate_scheduled( manager: KVCacheManager, request: Request, @@ -279,327 +175,6 @@ def _allocate_scheduled( return blocks -def _resident_block_ids(manager: KVCacheManager, request_id: str) -> list[int]: - """The GPU block HiSparse reads each resident page from; 0 if host-only.""" - resident = get_hisparse_coordinator(manager).resident_managers[0] - return [block.block_id for block in resident.get_residency_row(request_id)] - - -def test_hisparse_does_not_write_back_reprefillable_tokens(): - """Multi-module MTP re-prefills the last tokens, so they are not final.""" - manager = make_hisparse_kv_cache_manager(32, 16, num_prefill_lookahead=3) - request = make_request( - "request", list(range(2 * HISPARSE_BLOCK_SIZE)), HISPARSE_BLOCK_SIZE, sha256 - ) - assert _allocate_scheduled(manager, request, 2 * HISPARSE_BLOCK_SIZE) - transfers = get_hisparse_coordinator(manager).build_offload_command().page_transfers - assert len(transfers) == 1 - - -def test_hisparse_builds_dma_row_mirrors_across_pages(): - manager = make_hisparse_kv_cache_manager(32, 16) - request = make_request( - "request", - list(range(2 * HISPARSE_BLOCK_SIZE)), - HISPARSE_BLOCK_SIZE, - sha256, - ) - assert _allocate_scheduled(manager, request, num_new_tokens=32) is not None - coordinator = get_hisparse_coordinator(manager) - resident_blocks = coordinator.resident_managers[0].req_to_blocks[request.request_id] - host_blocks = coordinator.host_manager.req_to_blocks[request.request_id] - - mirrors = coordinator.build_row_mirrors([(request.request_id, 14, 4)]) - - assert [mirror.source_starts for mirror in mirrors] == [ - (resident_blocks[0].block_id * HISPARSE_BLOCK_SIZE + 14,), - (resident_blocks[1].block_id * HISPARSE_BLOCK_SIZE,), - ] - assert [mirror.destination_start for mirror in mirrors] == [ - host_blocks[0].block_id * HISPARSE_BLOCK_SIZE + 14, - host_blocks[1].block_id * HISPARSE_BLOCK_SIZE, - ] - assert [mirror.num_rows for mirror in mirrors] == [2, 2] - - -def test_hisparse_async_speculation_mirrors_uncertain_position_range(): - """Unresolved drafts must not leave gaps in the eager host mirror.""" - coordinator = MagicMock(host_group_id=0) - coordinator.take_residency_updates.return_value = {} - coordinator.build_offload_command.return_value = None - coordinator.build_row_mirrors.return_value = () - scheduler = HiSparseConnectorScheduler( - async_speculative=True, - draft_kv_lookahead=4, - ) - scheduler.bind_coordinator(coordinator) - request = SimpleNamespace(request_id="request", num_output_placeholders=3) - connector = object.__new__(HiSparseConnector) - connector.connector_scheduler = scheduler - connector.update_state_after_alloc(request, None, 0) - scheduler_output = SimpleNamespace( - kv_cache_block_copies=None, - scheduled_new_reqs=[], - scheduled_cached_reqs=SimpleNamespace( - new_block_ids=[], - req_ids=["request"], - num_computed_tokens=[103], - ), - num_scheduled_tokens={"request": 4}, - ) - - scheduler.build_connector_meta(scheduler_output) - - coordinator.build_row_mirrors.assert_called_once_with((("request", 100, 11),)) - - # Acceptance arrives between steps; the connector must read the live count. - request.num_output_placeholders = 1 - scheduler.build_connector_meta(scheduler_output) - coordinator.build_row_mirrors.assert_called_with((("request", 102, 9),)) - - connector.request_finished_all_groups(request, ()) - assert not scheduler.requests - - -def test_hisparse_reports_when_context_is_fully_resident(): - """A page the pool reuses under a host-reading request stops being resident.""" - manager = make_hisparse_kv_cache_manager(32, 16) - tokens = list(range(4 * HISPARSE_BLOCK_SIZE)) - request = make_request("request", tokens, HISPARSE_BLOCK_SIZE, sha256) - assert _allocate_scheduled(manager, request, num_new_tokens=len(tokens)) is not None - coordinator = get_hisparse_coordinator(manager) - scheduled = ((request.request_id, len(tokens) - 1, 1),) - assert coordinator.all_context_pages_resident(scheduled) - - for hot_manager in coordinator.hot_managers: - hot_manager.require_hot(request.request_id) - assert _allocate_scheduled(manager, request, num_new_tokens=len(tokens)) is not None - _publish_hisparse_pages(manager) - coordinator.take_residency_updates([request.request_id]) - core_row = manager.get_block_ids(request.request_id)[2] - pool = manager.block_pool - pool.get_new_blocks(pool.get_num_free_blocks()) - - assert not coordinator.all_context_pages_resident(scheduled) - # The block-table row stays append-only; the worker learns of the lost - # pages from the residency update instead. - assert manager.get_block_ids(request.request_id)[2] == core_row - resident_ids = _resident_block_ids(manager, request.request_id) - lost = [page for page, block_id in enumerate(resident_ids) if block_id == 0] - assert lost - update = coordinator.take_residency_updates([request.request_id]) - assert update[request.request_id].pages == lost - assert update[request.request_id].block_ids[0] == [0] * len(lost) - - -def test_hisparse_residency_update_sends_only_changed_pages(): - """Losing one early page must not resend the pages after it: per-step - residency updates would otherwise grow with the context length.""" - manager = make_hisparse_kv_cache_manager(32, 16) - tokens = list(range(6 * HISPARSE_BLOCK_SIZE)) - request = make_request("request", tokens, HISPARSE_BLOCK_SIZE, sha256) - assert _allocate_scheduled(manager, request, num_new_tokens=len(tokens)) - coordinator = get_hisparse_coordinator(manager) - for hot_manager in coordinator.hot_managers: - hot_manager.require_hot(request.request_id) - assert _allocate_scheduled(manager, request, num_new_tokens=len(tokens)) - coordinator.take_residency_updates([request.request_id]) - pool = manager.block_pool - num_never_used = pool.get_num_free_blocks() - # Pages unpin as their host copies complete, so page 0 is reused first. - _publish_hisparse_pages(manager) - pool.get_new_blocks(num_never_used + 1) - - update = coordinator.take_residency_updates([request.request_id]) - - assert update[request.request_id].pages == [0] - assert update[request.request_id].block_ids == ([0],) - - -def test_hisparse_readmitted_request_resends_its_full_residency(): - """A preempted request may resume in a worker state row holding another - request's residency, so its first update must cover every page again.""" - manager = make_hisparse_kv_cache_manager(32, 16) - coordinator = get_hisparse_coordinator(manager) - tokens = list(range(2 * HISPARSE_BLOCK_SIZE)) - request = make_request("request", tokens, HISPARSE_BLOCK_SIZE, sha256) - assert _allocate_scheduled(manager, request, len(tokens)) is not None - assert request.request_id in coordinator.take_residency_updates( - [request.request_id] - ) - manager.free(request) - - assert _allocate_scheduled(manager, request, len(tokens)) is not None - update = coordinator.take_residency_updates([request.request_id]) - assert update[request.request_id].pages == [0, 1] - assert update[request.request_id].block_ids[0] == _resident_block_ids( - manager, request.request_id - ) - - -def test_hisparse_host_prefix_can_be_completed_by_indexer_offload(): - """Restore the final host page when completing an indexer-only import.""" - manager = make_hisparse_kv_cache_manager( - 32, - 16, - enable_caching=True, - ) - tokens = list(range(4 * HISPARSE_BLOCK_SIZE)) - original = make_request("original", tokens, HISPARSE_BLOCK_SIZE, sha256) - assert ( - _allocate_scheduled(manager, original, num_new_tokens=len(tokens)) is not None - ) - spills = get_hisparse_coordinator(manager).build_offload_command().page_transfers - spill_counts = {spill.transfer_id: 1 for spill in spills} - get_hisparse_coordinator(manager).update_spills(spill_counts, spill_counts) - _, indexer_blocks, _, _ = manager.get_blocks(original.request_id).blocks - evicted_indexer_id = indexer_blocks[2].block_id - manager.free(original) - manager.block_pool.evict_blocks({evicted_indexer_id}) - - resumed = make_request("resumed", tokens, HISPARSE_BLOCK_SIZE, sha256) - blocks, num_local, _ = manager.get_computed_blocks(resumed) - max_completion = HISPARSE_BLOCK_SIZE - assert num_local == 2 * HISPARSE_BLOCK_SIZE - assert [len(group_blocks) for group_blocks in blocks.blocks] == [3, 2, 0, 0] - - allocated = _allocate_scheduled( - manager, - resumed, - num_new_tokens=1, - num_new_computed_tokens=num_local, - new_computed_blocks=blocks, - num_external_computed_tokens=max_completion, - ) - - assert allocated is not None - source, indexer, resident, hot = manager.get_blocks(resumed.request_id).blocks - assert len(source) == len(indexer) == len(resident) == 4 - assert len(hot) == 2 - assert 0 not in _resident_block_ids(manager, resumed.request_id) - coordinator = get_hisparse_coordinator(manager) - assert not coordinator.build_offload_command().page_transfers - coordinator.finish_host_import(resumed.request_id, failed=False) - transfers = coordinator.build_offload_command().page_transfers - assert len(transfers) == 1 and transfers[0].restore - assert transfers[0].resident_block_ids == (resident[2].block_id,) - - -def test_hisparse_indexer_offload_is_capped_by_missing_host_prefix(): - manager = make_hisparse_kv_cache_manager( - 32, - 16, - enable_caching=True, - ) - tokens = list(range(4 * HISPARSE_BLOCK_SIZE)) - original = make_request("original", tokens, HISPARSE_BLOCK_SIZE, sha256) - assert ( - _allocate_scheduled(manager, original, num_new_tokens=len(tokens)) is not None - ) - spills = get_hisparse_coordinator(manager).build_offload_command().page_transfers - spill_counts = {spill.transfer_id: 1 for spill in spills} - get_hisparse_coordinator(manager).update_spills(spill_counts, spill_counts) - host_blocks, _, _, _ = manager.get_blocks(original.request_id).blocks - evicted_host_id = host_blocks[0].block_id - manager.free(original) - get_hisparse_coordinator(manager).host_manager.block_pool.evict_blocks( - {evicted_host_id} - ) - - resumed = make_request("resumed", tokens, HISPARSE_BLOCK_SIZE, sha256) - _, num_local, _ = manager.get_computed_blocks(resumed) - connector = object.__new__(OffloadingConnector) - connector._kv_cache_config = manager.kv_cache_config - connector.bind_kv_cache_manager(manager) - assert num_local == 0 - assert connector._max_loadable_tokens(resumed, num_local) == 0 - - -@pytest.mark.parametrize("nested", [False, True]) -@pytest.mark.parametrize( - "available_tokens,imported_tokens,host_blocks", - [(0, 0, 2), (16, 16, 2), (64, 32, 3)], -) -def test_connector_completes_partial_prefix_without_importing_missing_host_kv( - nested, available_tokens, imported_tokens, host_blocks -): - manager = make_hisparse_kv_cache_manager(32, 16, enable_caching=True) - tokens = list(range(64)) - original = make_request("original", tokens, HISPARSE_BLOCK_SIZE, sha256) - assert _allocate_scheduled(manager, original, num_new_tokens=64) is not None - _publish_hisparse_pages(manager) - host, indexer, _, _ = manager.get_blocks(original.request_id).blocks - missing_indexer = indexer[1].block_id - missing_host = host[0].block_id - manager.free(original) - manager.block_pool.evict_blocks({missing_indexer}) - - defer = True - - def lookup(request, num_computed_tokens, max_num_new_tokens=None): - nonlocal defer - if defer: - defer = False - return None, False - available = available_tokens - if max_num_new_tokens is not None: - available = min(available, max_num_new_tokens) - return available, available > 0 - - connector = object.__new__(OffloadingConnector) - connector._kv_cache_config = manager.kv_cache_config - connector.connector_scheduler = SimpleNamespace(get_num_new_matched_tokens=lookup) - if nested: - for _ in range(2): - outer = object.__new__(MultiConnector) - outer._connectors = (connector,) - outer._requests_to_connector = {} - connector = outer - connector.bind_kv_cache_manager(manager) - scheduler = SimpleNamespace(connector=connector, kv_cache_manager=manager) - resumed = make_request("resumed", tokens, HISPARSE_BLOCK_SIZE, sha256) - - blocks, local, _, diverged = Scheduler._get_local_prefix_cache_hit( - scheduler, resumed - ) - assert local == 16 and not diverged - assert connector.get_num_new_matched_tokens(resumed, local) == (None, False) - - blocks, local, _, diverged = Scheduler._get_local_prefix_cache_hit( - scheduler, resumed - ) - external, asynchronous = connector.get_num_new_matched_tokens(resumed, local) - assert external == imported_tokens - assert asynchronous == (external > 0) - assert [len(group) for group in blocks.blocks] == [3, 1, 0, 0] - assert ( - _allocate_scheduled( - manager, - resumed, - int(external == 0), - delay_cache_blocks=True, - num_new_computed_tokens=local, - new_computed_blocks=blocks, - num_external_computed_tokens=external, - ) - is not None - ) - allocated = manager.get_blocks(resumed.request_id).blocks - assert len(allocated[0]) == host_blocks - assert allocated[2][1].is_null == (host_blocks > 2) - assert not allocated[2][-1].is_null - manager.free(resumed) - - # A surviving indexer offload must not make absent host KV look computed. - get_hisparse_coordinator(manager).host_manager.block_pool.evict_blocks( - {missing_host} - ) - retry = make_request("retry", tokens, HISPARSE_BLOCK_SIZE, sha256) - _, local, _, _ = Scheduler._get_local_prefix_cache_hit(scheduler, retry) - assert local == 0 - assert connector.get_num_new_matched_tokens(retry, local) == (0, False) - - def allocate_external_prefix( manager: KVCacheManager, request: Request, num_tokens: int ) -> KVCacheBlocks | None: @@ -613,595 +188,6 @@ def allocate_external_prefix( ) -@pytest.mark.parametrize("num_tokens", [1, 15, 16, 17, 31, 32, 33, 127]) -def test_nixl_hisparse_full_block_import_keeps_a_writable_tail(num_tokens): - """Import every real token and restore the page used by last-token replay.""" - from tests.v1.kv_connector.unit.utils import create_vllm_config - from vllm.distributed.kv_transfer.kv_connector.v1.nixl.pull_scheduler import ( - NixlPullConnectorScheduler, - ) - - manager = make_hisparse_kv_cache_manager(32, 16, enable_caching=True) - config = create_vllm_config(block_size=HISPARSE_BLOCK_SIZE) - connector = NixlPullConnectorScheduler(config, "test", manager.kv_cache_config) - request = make_request( - "import", list(range(num_tokens)), HISPARSE_BLOCK_SIZE, sha256 - ) - request.kv_transfer_params = {"do_remote_prefill": True} - count, is_async = connector.get_num_new_matched_tokens(request, 0) - assert count == num_tokens and is_async - assert allocate_external_prefix(manager, request, count) is not None - source, _, resident, _ = manager.get_blocks(request.request_id).blocks - assert len(source) == len(resident) == (num_tokens + 15) // 16 - assert all(block.is_null for block in resident[:-1]) - assert not resident[-1].is_null - coordinator = get_hisparse_coordinator(manager) - assert not coordinator.build_offload_command().page_transfers - - coordinator.finish_host_import(request.request_id, failed=False) - transfers = coordinator.build_offload_command().page_transfers - assert len(transfers) == 1 - restore = transfers[0] - assert restore.restore and not restore.after_forward - assert restore.host_block_id == source[-1].block_id - assert restore.resident_block_ids == (resident[-1].block_id,) - request.num_computed_tokens = count - scheduler = SimpleNamespace( - connector=connector, - kv_cache_manager=manager, - failed_recving_kv_req_ids=set(), - finished_recving_kv_req_ids={request.request_id}, - prefix_replay_tokens=0, - prefix_replay_group_ids=(), - ) - scheduler._mark_prefix_replay = MethodType(Scheduler._mark_prefix_replay, scheduler) - scheduler._load_restores_replay_window = MethodType( - Scheduler._load_restores_replay_window, scheduler - ) - Scheduler._update_waiting_for_remote_kv(scheduler, request) - assert request.num_tokens - request.num_computed_tokens == 1 - assert _allocate_scheduled(manager, request, num_new_tokens=1) is not None - assert coordinator.build_row_mirrors([(request.request_id, num_tokens - 1, 1)]) - # A pending restore must not expose uninitialized GPU copies to prefix hits. - assert all(resident[-1] not in copies for copies in coordinator.copies.values()) - - coordinator.update_spills({restore.transfer_id: 2}, {restore.transfer_id: 1}) - assert resident[-1].ref_cnt == 2 - assert all(resident[-1] not in copies for copies in coordinator.copies.values()) - coordinator.update_spills({}, {restore.transfer_id: 1}) - assert resident[-1].ref_cnt == 1 - assert coordinator.request_states[request.request_id].valid_pages == set( - range(num_tokens // HISPARSE_BLOCK_SIZE) - ) - if num_tokens % HISPARSE_BLOCK_SIZE: - assert source[-1].block_hash is None - else: - assert coordinator.copies[source[-1].block_hash] == (resident[-1],) - - -def test_hisparse_aborted_tail_restore_retains_both_endpoints(): - """An abort cannot recycle storage still used by a queued tail restore.""" - manager = make_hisparse_kv_cache_manager(16, 16) - request = make_request("import", list(range(17)), HISPARSE_BLOCK_SIZE, sha256) - assert allocate_external_prefix(manager, request, 17) is not None - source, _, resident, _ = manager.get_blocks(request.request_id).blocks - host_tail, gpu_tail = source[-1], resident[-1] - coordinator = get_hisparse_coordinator(manager) - coordinator.finish_host_import(request.request_id, failed=False) - restore = coordinator.build_offload_command().page_transfers[0] - manager.free(request) - assert host_tail.ref_cnt == gpu_tail.ref_cnt == 1 - coordinator.update_spills({restore.transfer_id: 2}, {restore.transfer_id: 1}) - assert host_tail.ref_cnt == gpu_tail.ref_cnt == 1 - coordinator.update_spills({}, {restore.transfer_id: 1}) - assert host_tail.ref_cnt == gpu_tail.ref_cnt == 0 - assert not coordinator.has_pending_work() - - -@pytest.mark.parametrize("enable_caching", [False, True]) -def test_hisparse_keeps_resident_pages_until_hot_buffer_is_allocated(enable_caching): - """Later admissions must not remove a scheduled request's only readable tier.""" - manager = make_hisparse_kv_cache_manager( - 32, 32, max_model_len=256, enable_caching=enable_caching - ) - first = make_request("first", list(range(160)), HISPARSE_BLOCK_SIZE, sha256) - assert _allocate_scheduled(manager, first, num_new_tokens=128) is not None - _publish_hisparse_pages(manager) - first.num_computed_tokens = 128 - pool = manager.block_pool - held = pool.get_new_blocks(pool.get_num_free_blocks() - 3) - - assert _allocate_scheduled(manager, first, num_new_tokens=1) is not None - second = make_request("second", [1000] * 16, HISPARSE_BLOCK_SIZE, sha256) - _allocate_scheduled(manager, second, num_new_tokens=16) - coordinator = get_hisparse_coordinator(manager) - scheduled = (("first", 128, 1),) - assert coordinator.all_context_pages_resident(scheduled) - assert all(not m.has_hot("first") for m in coordinator.hot_managers) - - # Once another allocation finishes, A can acquire its hot buffer and - # safely release its clean pages for other requests to reuse. - pool.free_blocks(held) - assert _allocate_scheduled(manager, first, num_new_tokens=1) is not None - assert all(m.has_hot("first") for m in coordinator.hot_managers) - coordinator.take_residency_updates(["first"]) - pool.get_new_blocks(pool.get_num_free_blocks()) - assert not coordinator.all_context_pages_resident(scheduled) - assert "first" in coordinator.take_residency_updates(["first"]) - - -def test_hisparse_full_pool_keeps_pages_pinned_until_preemption(): - """Without room for a hot buffer, admission must defer instead of losing KV.""" - manager = make_hisparse_kv_cache_manager(18, 18, max_model_len=160) - first = make_request("first", list(range(128)), HISPARSE_BLOCK_SIZE, sha256) - assert _allocate_scheduled(manager, first, num_new_tokens=128) is not None - _publish_hisparse_pages(manager) - assert manager.block_pool.get_num_free_blocks() == 1 - first.num_computed_tokens = 128 - assert _allocate_scheduled(manager, first, num_new_tokens=1) is None - assert get_hisparse_coordinator(manager).all_context_pages_resident( - (("first", 127, 1),) - ) - - manager.free(first) - second = make_request("second", [1000] * 16, HISPARSE_BLOCK_SIZE, sha256) - assert _allocate_scheduled(manager, second, num_new_tokens=16) is not None - - -@pytest.mark.parametrize("num_resident_groups", [1, 3]) -def test_hisparse_materializes_prefix_without_allocating_hot_blocks( - num_resident_groups, -): - """A host prefix becomes visible only when every page is durable. - - Completing a later page first must not expose a prefix with a hole. - """ - manager = make_hisparse_kv_cache_manager( - 32, - 16, - enable_caching=True, - num_resident_groups=num_resident_groups, - ) - tokens = list(range(2 * HISPARSE_BLOCK_SIZE)) - request = make_request("resident", tokens, HISPARSE_BLOCK_SIZE, sha256) - assert _allocate_scheduled(manager, request, num_new_tokens=len(tokens)) is not None - blocks = manager.get_block_ids(request.request_id) - assert len(blocks[2]) == 2 - assert blocks[3] == [] - - spills = get_hisparse_coordinator(manager).build_offload_command().page_transfers - assert len(spills) == 2 - assert all(spill.after_forward for spill in spills) - resident_blocks = manager.get_blocks(request.request_id).blocks[2] - assert [block.ref_cnt for block in resident_blocks] == [2, 2] - spill_counts = {spill.transfer_id: 1 for spill in spills} - duplicate = make_request( - "duplicate", - list(range(3 * HISPARSE_BLOCK_SIZE)), - HISPARSE_BLOCK_SIZE, - sha256, - ) - _, num_computed, _ = manager.get_computed_blocks(duplicate) - assert num_computed == 0 - - get_hisparse_coordinator(manager).update_spills(spill_counts, {}) - assert [block.ref_cnt for block in resident_blocks] == [2, 2] - _, num_computed, _ = manager.get_computed_blocks(duplicate) - assert num_computed == 0 - - get_hisparse_coordinator(manager).update_spills({}, {spills[1].transfer_id: 1}) - assert [block.ref_cnt for block in resident_blocks] == [2, 1] - _, num_computed, _ = manager.get_computed_blocks(duplicate) - assert num_computed == 0 - - get_hisparse_coordinator(manager).update_spills({}, {spills[0].transfer_id: 1}) - _, num_computed, _ = manager.get_computed_blocks(duplicate) - assert num_computed == 2 * HISPARSE_BLOCK_SIZE - blocks = manager.get_block_ids(request.request_id) - assert len(blocks[2]) == 2 - assert blocks[3] == [] - - -@pytest.mark.parametrize("num_resident_groups", [1, 3]) -def test_hisparse_materialization_respects_per_step_spill_budget(num_resident_groups): - """Prefix publication must not bypass the configured spill batch limit.""" - manager = make_hisparse_kv_cache_manager( - 32, - 16, - enable_caching=True, - num_resident_groups=num_resident_groups, - ) - coordinator = get_hisparse_coordinator(manager) - coordinator.max_spill_pages = 1 - tokens = list(range(2 * HISPARSE_BLOCK_SIZE)) - request = make_request("bounded", tokens, HISPARSE_BLOCK_SIZE, sha256) - - assert _allocate_scheduled(manager, request, num_new_tokens=len(tokens)) is not None - first = coordinator.build_offload_command().page_transfers - assert len(first) == 1 - - coordinator.plan_prefix_materialization(request.request_id, len(tokens)) - second = coordinator.build_offload_command().page_transfers - assert len(second) == 1 - assert first[0].transfer_id != second[0].transfer_id - - -def test_hisparse_deferred_free_retains_host_blocks_until_fence(): - """Detaching a request must not make in-flight host pages reusable.""" - manager = make_hisparse_kv_cache_manager(16, 16) - host_pool = get_hisparse_coordinator(manager).get_host_block_pool() - assert host_pool is not None - host_free = host_pool.get_num_free_blocks() - device_free = manager.block_pool.get_num_free_blocks() - request = make_request("free", list(range(16)), HISPARSE_BLOCK_SIZE, sha256) - assert ( - _allocate_scheduled(manager, request, 16, delay_cache_blocks=True) is not None - ) - blocks = manager.pop_blocks_for_free(request) - assert host_pool.get_num_free_blocks() < host_free - assert blocks and any(host_pool.blocks[b.block_id] is b for b in blocks) - manager.block_pool.free_blocks(reversed(blocks)) - assert host_pool.get_num_free_blocks() == host_free - assert manager.block_pool.get_num_free_blocks() == device_free - - -def test_hisparse_host_cow_copy_is_drained_without_a_gpu_pool(): - """Host-only copy-on-write work must reach the worker copy queue.""" - manager = make_hisparse_kv_cache_manager(16, 16) - source_manager = manager.coordinator.single_type_managers[0] - source_block = source_manager.block_pool.get_new_blocks(1)[0] - source_manager.req_to_blocks["cow"] = [source_block] - source_manager._partial_hit_reqs["cow"] = (0, source_block) - - new_blocks = source_manager.allocate_new_blocks( - "cow", HISPARSE_BLOCK_SIZE, HISPARSE_BLOCK_SIZE - ) - new_block_ids = manager.take_new_block_ids() - coordinator = get_hisparse_coordinator(manager) - assert manager.take_kv_cache_block_copies() == ([], []) - connector = HiSparseConnectorScheduler(async_speculative=False) - connector.bind_coordinator(coordinator) - output = SchedulerOutput.make_empty() - copies = connector.build_connector_meta(output).host_block_copies - assert output.has_sync_kv_loads - - assert new_blocks and new_block_ids == [] - assert len(copies) == 1 - assert copies[0].src_block_id == source_block.block_id - assert copies[0].dst_block_id == new_blocks[0].block_id - source_manager.free("cow") - assert source_block.ref_cnt > 0 and new_blocks[0].ref_cnt > 0 - assert coordinator.has_pending_work() - connector.update_connector_output( - KVConnectorOutput( - kv_connector_worker_meta=HiSparseConnectorWorkerMetadata( - {}, {}, completed_host_copy_dst_ids=(copies[0].dst_block_id,) - ) - ) - ) - assert source_block.ref_cnt == new_blocks[0].ref_cnt == 0 - assert not coordinator.has_pending_work() - - -def test_hisparse_inflight_host_import_reserves_remaining_gpu_pages(): - """An in-flight host import must reserve its unwritten resident pages.""" - manager = make_hisparse_kv_cache_manager( - 12, - 16, - transfer_device_cache=True, - ) - request = make_request( - "partial-import", - list(range(6 * HISPARSE_BLOCK_SIZE)), - HISPARSE_BLOCK_SIZE, - sha256, - ) - imported_tokens = 4 * HISPARSE_BLOCK_SIZE - - assert allocate_external_prefix(manager, request, imported_tokens) is not None - required = manager.coordinator.get_num_blocks_to_allocate( - request_id=request.request_id, - num_tokens=request.num_tokens, - new_computed_blocks=manager.empty_kv_cache_blocks.blocks, - num_encoder_tokens=0, - total_computed_tokens=imported_tokens, - num_local_computed_tokens=0, - num_tokens_main_model=request.num_tokens, - apply_admission_cap=True, - ) - - assert required == 4 - - -@pytest.mark.parametrize("host_num_blocks,admitted", [(5, False), (6, True)]) -def test_hisparse_async_admission_requires_full_prompt_host_pages( - host_num_blocks, admitted, tmp_path -): - """An async load is admitted only if its whole prompt's host pages fit. - - The in-flight prefill's host page is not free, so the waiting request's - four prompt pages need six host blocks including the null block. - """ - from .utils import create_scheduler, mock_kv - - (tmp_path / "config.json").write_text( - '{"architectures": ["OPTForCausalLM"], "model_type": "opt"}' - ) - scheduler = create_scheduler( - model=str(tmp_path), - skip_tokenizer_init=True, - max_model_len=128, - use_kv_connector=mock_kv(matched_tokens=HISPARSE_BLOCK_SIZE, is_async=True), - ) - manager = make_hisparse_kv_cache_manager(32, host_num_blocks) - scheduler.kv_cache_manager = manager - scheduler.kv_cache_config = manager.kv_cache_config - assert scheduler.scheduler_reserve_full_isl - inflight = make_request( - "inflight", list(range(3 * HISPARSE_BLOCK_SIZE)), HISPARSE_BLOCK_SIZE, sha256 - ) - scheduler.add_request(inflight) - scheduler.schedule() - assert inflight in scheduler._inflight_prefills - - request = make_request( - "waiting", list(range(4 * HISPARSE_BLOCK_SIZE)), HISPARSE_BLOCK_SIZE, sha256 - ) - scheduler.add_request(request) - scheduler.schedule() - - assert (request in scheduler._inflight_prefills) == admitted - assert bool(manager.get_blocks(request.request_id).blocks[0]) == admitted - - -def test_hisparse_admitted_async_loads_can_finish_with_nothing_running(tmp_path): - """Admitted async loads must not strand each other on host capacity. - - Waiting requests keep their blocks and are never preempted, so two imports - that each hold a host page while together needing more than the pool has - left would never be scheduled again. HiSparse relies on full-ISL admission: - a load is admitted only if its whole prompt's host pages fit, so the newest - admitted load can always finish and free host blocks for the others. - """ - from .utils import create_scheduler, mock_kv - - (tmp_path / "config.json").write_text( - '{"architectures": ["OPTForCausalLM"], "model_type": "opt"}' - ) - scheduler = create_scheduler( - model=str(tmp_path), - skip_tokenizer_init=True, - max_model_len=128, - use_kv_connector=mock_kv(matched_tokens=HISPARSE_BLOCK_SIZE, is_async=True), - ) - # One null block plus three usable host blocks. - manager = make_hisparse_kv_cache_manager(32, 4) - scheduler.kv_cache_manager = manager - scheduler.kv_cache_config = manager.kv_cache_config - assert scheduler.scheduler_reserve_full_isl - first = make_request( - "first", list(range(3 * HISPARSE_BLOCK_SIZE)), HISPARSE_BLOCK_SIZE, sha256 - ) - second = make_request( - "second", - list(range(1000, 1000 + 3 * HISPARSE_BLOCK_SIZE)), - HISPARSE_BLOCK_SIZE, - sha256, - ) - for request in (first, second): - scheduler.add_request(request) - scheduler.schedule() - admitted = {r.request_id for r in scheduler._inflight_prefills} - - scheduler.finished_recving_kv_req_ids.update(admitted) - num_scheduled_tokens = 0 - for _ in range(4): - num_scheduled_tokens += scheduler.schedule().total_num_scheduled_tokens - assert num_scheduled_tokens > 0 - - -@pytest.mark.parametrize( - "evictable,local_tokens,free_blocks,admitted", - [ - (False, HISPARSE_BLOCK_SIZE, 1, True), - (True, HISPARSE_BLOCK_SIZE, 1, False), - (False, HISPARSE_BLOCK_SIZE - 1, 1, False), - (False, HISPARSE_BLOCK_SIZE - 1, 2, True), - ], -) -def test_hisparse_import_capacity_includes_hits_and_cow( - evictable, local_tokens, free_blocks, admitted -): - """Touching an evictable hit and copying a partial hit consume capacity.""" - manager = make_hisparse_kv_cache_manager(32, free_blocks + 2 - int(evictable)) - coordinator = get_hisparse_coordinator(manager) - host_pool = coordinator.get_host_block_pool() - assert host_pool is not None - hit = host_pool.get_new_blocks(1)[0] - if evictable: - host_pool.free_blocks([hit]) - assert host_pool.get_num_free_blocks() == free_blocks - assert ( - coordinator.host_manager.get_num_blocks_to_allocate( - "waiting", 32, [hit], 32, local_tokens, 32 - ) - <= host_pool.get_num_free_blocks() - ) == admitted - - -@pytest.mark.parametrize("enable_caching", [False, True]) -def test_hisparse_host_exhaustion_defers_allocation(enable_caching): - """Pages are only allocated with host backing, so every one can be reclaimed.""" - manager = make_hisparse_kv_cache_manager(32, 5, enable_caching=enable_caching) - donor = make_request("donor", [99] * 16, HISPARSE_BLOCK_SIZE, sha256) - assert _allocate_scheduled(manager, donor, 16) is not None - request = make_request("resident", list(range(64)), HISPARSE_BLOCK_SIZE, sha256) - free_gpu = manager.block_pool.get_num_free_blocks() - assert manager.allocate_slots(request, 64) is None - assert manager.block_pool.get_num_free_blocks() == free_gpu - - # The donor's in-flight write-back holds its host block until acked. - manager.free(donor) - coordinator = get_hisparse_coordinator(manager) - assert coordinator.has_pending_reclamation() - assert manager.allocate_slots(request, 64) is None - _publish_hisparse_pages(manager) - assert not coordinator.has_pending_reclamation() - assert _allocate_scheduled(manager, request, 64) is not None - host = coordinator.host_manager - assert host is not None - assert not any(b.is_null for b in host.req_to_blocks[request.request_id]) - _publish_hisparse_pages(manager) - for hot in coordinator.hot_managers: - hot.require_hot(request.request_id) - hot.allocate_new_blocks(request.request_id, 64, 64) - coordinator.update_residency(request.request_id) - assert coordinator.request_states[request.request_id].unpinned_pages == {0, 1} - for resident in coordinator.resident_managers: - assert [b.ref_cnt for b in resident.req_to_blocks[request.request_id]] == [ - 0, - 0, - 1, - 1, - ] - - -@pytest.mark.parametrize("failed", [False, True]) -def test_host_receive_completion_without_spill_metadata(failed): - """Only successful external receives make host pages readable.""" - manager = make_hisparse_kv_cache_manager(32, 16) - request = make_request("import", list(range(32)), HISPARSE_BLOCK_SIZE, sha256) - assert allocate_external_prefix(manager, request, 32) is not None - request.num_computed_tokens = 32 - request.status = RequestStatus.WAITING_FOR_REMOTE_KVS - scheduler = HiSparseConnectorScheduler(async_speculative=False) - scheduler.bind_coordinator(get_hisparse_coordinator(manager)) - scheduler.requests[request.request_id] = request - connector = object.__new__(HiSparseConnector) - connector.connector_scheduler = scheduler - composite = object.__new__(MultiConnector) - composite._connectors = (connector,) - composite.update_connector_output( - KVConnectorOutput( - finished_recving={request.request_id}, - failed_recving={request.request_id} if failed else set(), - ) - ) - state = get_hisparse_coordinator(manager).request_states.get(request.request_id) - if failed: - assert state is None - else: - assert state is not None and state.valid_pages == {0, 1} - - -def test_hisparse_recomputation_does_not_complete_failed_host_import(): - """Allocating recomputation is not evidence that imported host KV is valid.""" - manager = make_hisparse_kv_cache_manager(32, 16, enable_caching=True) - request = make_request( - "partial-failure", list(range(48)), HISPARSE_BLOCK_SIZE, sha256 - ) - assert allocate_external_prefix(manager, request, 32) is not None - # A partial load failure leaves only the first page computed. The next - # allocation reaches the original import boundary before the worker runs. - request.num_computed_tokens = 16 - manager.cache_blocks(request, 16) - assert _allocate_scheduled(manager, request, 16) is not None - state = get_hisparse_coordinator(manager).request_states.get(request.request_id) - assert state is None or 1 not in state.valid_pages - - -def test_hisparse_host_import_ignores_unsealed_tail(): - """A partial imported page must not become readable from stale host data.""" - manager = make_hisparse_kv_cache_manager(16, 16) - coordinator = get_hisparse_coordinator(manager) - - coordinator.complete_host_import("partial", HISPARSE_BLOCK_SIZE + 1) - - state = coordinator.request_states["partial"] - assert state.valid_pages == {0} - assert state.ready_prefix_pages == 1 - - -def test_hisparse_resident_request_can_grow_without_hot_capacity(): - """Running resident history must not be mistaken for a new host-prefix hit.""" - manager = make_hisparse_kv_cache_manager(8, 16) - request = make_request("resident", list(range(48)), HISPARSE_BLOCK_SIZE, sha256) - assert _allocate_scheduled(manager, request, num_new_tokens=32) is not None - request.num_computed_tokens = 32 - - # Three free blocks fit the next indexer/resident pages, but not a hot region. - assert manager.block_pool.get_num_free_blocks() == 3 - assert _allocate_scheduled(manager, request, num_new_tokens=1) is not None - assert manager.get_block_ids(request.request_id)[3] == [] - - -def test_hisparse_capacity_query_does_not_require_hot_blocks(): - """A read-only capacity query must not mutate hot-block requirements.""" - manager = make_hisparse_kv_cache_manager(16, 16) - coordinator = get_hisparse_coordinator(manager) - host_pool = coordinator.get_host_block_pool() - assert host_pool is not None - computed: tuple[list[KVCacheBlock], ...] = ( - [host_pool.blocks[1]], - [], - [], - [], - ) - - manager.coordinator.get_num_blocks_to_allocate( - request_id="query-only", - num_tokens=HISPARSE_BLOCK_SIZE, - new_computed_blocks=computed, - num_encoder_tokens=0, - total_computed_tokens=HISPARSE_BLOCK_SIZE, - num_local_computed_tokens=HISPARSE_BLOCK_SIZE, - num_tokens_main_model=HISPARSE_BLOCK_SIZE, - ) - - assert all( - "query-only" not in hot_manager.hot_required - for hot_manager in coordinator.hot_managers - ) - - -def _publish_hisparse_pages(manager: KVCacheManager) -> None: - """Ack all planned spills so host pages publish to the prefix cache.""" - command = get_hisparse_coordinator(manager).build_offload_command() - counts = {spill.transfer_id: 1 for spill in command.page_transfers} - get_hisparse_coordinator(manager).update_spills(counts, counts) - - -def test_hisparse_events_report_host_and_device_placement(): - """Host publication and eviction must not advertise GPU-resident KV.""" - manager = make_kv_cache_manager( - make_hisparse_kv_cache_config(32, 16), - max_model_len=128, - enable_caching=True, - hash_block_size=HISPARSE_BLOCK_SIZE, - enable_kv_cache_events=True, - ) - request = make_request("request", list(range(64)), HISPARSE_BLOCK_SIZE, sha256) - assert _allocate_scheduled(manager, request, 64) is not None - _publish_hisparse_pages(manager) - - stored = [ - event for event in manager.take_events() if isinstance(event, BlockStored) - ] - assert {(event.group_idx, event.medium) for event in stored} == { - (0, MEDIUM_CPU), - (1, MEDIUM_GPU), - } - - host_block = manager.get_blocks(request.request_id).blocks[0][0] - get_hisparse_coordinator(manager).host_manager.block_pool.evict_blocks( - {host_block.block_id} - ) - [removed] = manager.take_events() - assert isinstance(removed, BlockRemoved) - assert removed.medium == MEDIUM_CPU - assert removed.group_idx == 0 - - def test_mamba_boundary_handoffs_do_not_pin_obsolete_blocks(): """Draining unused connector offers must not retain old recurrent states.""" manager = make_kv_cache_manager( @@ -1239,365 +225,6 @@ def test_mamba_boundary_handoffs_do_not_pin_obsolete_blocks(): assert all(block.ref_cnt == 0 for block in old_blocks[:-1]) -def test_hisparse_prefix_hit_adopts_gpu_shadow_pages(): - """A host prefix hit must come back GPU-resident while shadows survive.""" - manager = make_hisparse_kv_cache_manager(32, 16, enable_caching=True) - tokens = list(range(4 * HISPARSE_BLOCK_SIZE)) - original = make_request("original", tokens, HISPARSE_BLOCK_SIZE, sha256) - assert ( - _allocate_scheduled(manager, original, num_new_tokens=len(tokens)) is not None - ) - _publish_hisparse_pages(manager) - original_resident_ids = [ - block.block_id for block in manager.get_blocks("original").blocks[2] - ] - manager.free(original) - - resumed = make_request("resumed", tokens, HISPARSE_BLOCK_SIZE, sha256) - computed, num_computed, _ = manager.get_computed_blocks(resumed) - assert num_computed == 3 * HISPARSE_BLOCK_SIZE - assert _allocate_scheduled( - manager, - resumed, - num_new_tokens=len(tokens) - num_computed, - num_new_computed_tokens=num_computed, - new_computed_blocks=computed, - ) - assert _resident_block_ids(manager, "resumed")[:3] == original_resident_ids[:3] - assert get_hisparse_coordinator(manager).all_context_pages_resident( - ((resumed.request_id, num_computed, len(tokens) - num_computed),) - ) - - -@pytest.mark.parametrize("free_blocks,adopted_pages", [(7, []), (10, [0, 1])]) -def test_hisparse_prefix_hit_under_pressure_adopts_surviving_copies( - free_blocks, adopted_pages -): - """A host prefix hit must not starve its own allocation of free blocks. - - Admission counts free GPU copies as free, but adopting a copy pins it. When - adoption ran before the hit's allocation, it took blocks the allocation was - promised and the pool ran dry (``Cannot get N free blocks``). The allocation - now goes first, evicting a prefix's copies tail first; the copies it leaves - are adopted. - """ - manager = make_hisparse_kv_cache_manager(32, 16, enable_caching=True) - tokens = list(range(4 * HISPARSE_BLOCK_SIZE)) - original = make_request("original", tokens, HISPARSE_BLOCK_SIZE, sha256) - assert ( - _allocate_scheduled(manager, original, num_new_tokens=len(tokens)) is not None - ) - _publish_hisparse_pages(manager) - copy_ids = [block.block_id for block in manager.get_blocks("original").blocks[2]] - manager.free(original) - pool = manager.block_pool - pool.get_new_blocks(pool.get_num_free_blocks() - free_blocks) - - resumed = make_request("resumed", tokens, HISPARSE_BLOCK_SIZE, sha256) - computed, num_computed, _ = manager.get_computed_blocks(resumed) - assert num_computed == 3 * HISPARSE_BLOCK_SIZE - assert _allocate_scheduled( - manager, - resumed, - num_new_tokens=len(tokens) - num_computed, - num_new_computed_tokens=num_computed, - new_computed_blocks=computed, - ) - - resident_ids = _resident_block_ids(manager, "resumed") - assert resident_ids[:3] == [ - copy_ids[page] if page in adopted_pages else pool.null_block.block_id - for page in range(3) - ] - _, indexer, _, hot = manager.get_block_ids("resumed") - gpu_ids = [block_id for block_id in indexer + resident_ids + hot if block_id] - assert len(gpu_ids) == len(set(gpu_ids)) - - -def test_hisparse_host_backed_request_accepts_local_prefix_hit(): - """Local group-completion hits need no external host import allocation.""" - manager = make_hisparse_kv_cache_manager(32, 16, enable_caching=True) - tokens = list(range(4 * HISPARSE_BLOCK_SIZE)) - original = make_request("original", tokens, HISPARSE_BLOCK_SIZE, sha256) - assert ( - _allocate_scheduled(manager, original, num_new_tokens=len(tokens)) is not None - ) - _publish_hisparse_pages(manager) - manager.free(original) - - resumed = make_request("resumed", tokens, HISPARSE_BLOCK_SIZE, sha256) - computed, num_computed, _ = manager.get_computed_blocks(resumed) - assert num_computed == 3 * HISPARSE_BLOCK_SIZE - assert _allocate_scheduled( - manager, - resumed, - num_new_tokens=len(tokens) - num_computed, - num_new_computed_tokens=num_computed, - new_computed_blocks=computed, - num_external_computed_tokens=0, - ) - - -def test_hisparse_finished_request_leaves_free_copies(): - """Clean pages of a finished request are free-queue blocks, not pins.""" - manager = make_hisparse_kv_cache_manager(32, 16, enable_caching=True) - tokens = list(range(4 * HISPARSE_BLOCK_SIZE)) - original = make_request("original", tokens, HISPARSE_BLOCK_SIZE, sha256) - assert ( - _allocate_scheduled(manager, original, num_new_tokens=len(tokens)) is not None - ) - _publish_hisparse_pages(manager) - manager.free(original) - - coordinator = get_hisparse_coordinator(manager) - assert coordinator.copies - pool = manager.block_pool - assert pool.get_num_free_blocks() == pool.num_gpu_blocks - 1 - pool.get_new_blocks(pool.get_num_free_blocks()) - assert not coordinator.copies - assert not coordinator.spills_to_send - - -@pytest.mark.parametrize( - "status,expected_cached_pages", - [ - (RequestStatus.FINISHED_STOPPED, 2), - (RequestStatus.FINISHED_ABORTED, 2), - (RequestStatus.RUNNING, 0), - ], - ids=["stopped", "aborted", "running"], -) -def test_hisparse_cleanup_publishes_only_finalized_terminal_pages( - status, expected_cached_pages -): - """Only terminal cleanup may publish completed, finalized pages.""" - manager = make_hisparse_kv_cache_manager(32, 16, enable_caching=True) - coordinator = get_hisparse_coordinator(manager) - request = make_request("terminal", list(range(33)), HISPARSE_BLOCK_SIZE, sha256) - request.spec_token_ids = list(range(33, 49)) - assert ( - _allocate_scheduled(manager, request, 49, num_lookahead_tokens=16) is not None - ) - host_blocks = list(manager.get_blocks(request.request_id).blocks[0]) - command = coordinator.build_offload_command() - counts = {transfer.transfer_id: 1 for transfer in command.page_transfers} - assert len(counts) == 2 - request.status = status - manager.free(request) - coordinator.update_spills(counts, counts) - assert all( - block.block_hash is not None for block in host_blocks[:expected_cached_pages] - ) - assert all( - block.block_hash is None for block in host_blocks[expected_cached_pages:] - ) - assert all(block.ref_cnt == 0 for block in host_blocks) - assert not coordinator.has_pending_work() - - -@pytest.mark.parametrize("num_pages,reads_host", [(4, True), (3, False)]) -def test_hisparse_prefill_reads_host_once_it_fills_admission_window( - num_pages, reads_host -): - """A prefill switches to host reads once it fills its admission window. - - Admission caps each request's resident pages at the in-flight window, - assuming older pages move to host, but pinned pages were released only - once free blocks fell below a fixed watermark. A chunked prefill larger - than the watermark outgrew the pool first and was preempted. - """ - window_pages = 4 - manager = make_hisparse_kv_cache_manager( - 32, - 16, - enable_caching=True, - max_in_flight_tokens=window_pages * HISPARSE_BLOCK_SIZE, - ) - coordinator = get_hisparse_coordinator(manager) - tokens = list(range(2 * window_pages * HISPARSE_BLOCK_SIZE)) - request = make_request("prefill", tokens, HISPARSE_BLOCK_SIZE, sha256) - assert ( - _allocate_scheduled(manager, request, num_pages * HISPARSE_BLOCK_SIZE) - is not None - ) - pool = coordinator.gpu_pool - assert pool is not None - assert pool.get_num_free_blocks() >= coordinator.transition_watermark - - hot_manager = coordinator.hot_managers[0] - assert (request.request_id in hot_manager.hot_required) == reads_host - - -def test_hisparse_preempted_prefill_resumes_from_durable_prefix(): - """A prefill preempted mid-way must resume from the pages already on host. - - Host writes trail a chunked prefill by a chunk, and host pages were only - published once the whole computed prefix was durable. Each chunk moved - that target past the finished writes, so nothing was published before the - prefill ended: a prefill preempted each time it outgrew the GPU pool - recomputed from the start forever. - """ - manager = make_hisparse_kv_cache_manager(32, 16, enable_caching=True) - coordinator = get_hisparse_coordinator(manager) - chunk = 2 * HISPARSE_BLOCK_SIZE - tokens = list(range(3 * chunk)) - request = make_request("long", tokens, HISPARSE_BLOCK_SIZE, sha256) - assert _allocate_scheduled(manager, request, chunk) is not None - request.num_computed_tokens = chunk - first_chunk_writes = coordinator.build_offload_command().page_transfers - assert _allocate_scheduled(manager, request, chunk) is not None - request.num_computed_tokens = 2 * chunk - acks = {transfer.transfer_id: 1 for transfer in first_chunk_writes} - coordinator.update_spills(acks, acks) - manager.free(request) - - resumed = make_request("resumed", tokens, HISPARSE_BLOCK_SIZE, sha256) - assert manager.get_computed_blocks(resumed)[1] == chunk - - -def test_hisparse_terminal_prefix_waits_for_all_workers_and_preserves_identity(): - """Old completions retain their pages through pressure and request-ID reuse.""" - manager = make_hisparse_kv_cache_manager(32, 8, enable_caching=True) - coordinator = get_hisparse_coordinator(manager) - host_pool = coordinator.get_host_block_pool() - tokens = list(range(2 * HISPARSE_BLOCK_SIZE + 1)) - request = make_request("reused", tokens, HISPARSE_BLOCK_SIZE, sha256) - assert _allocate_scheduled(manager, request, len(tokens)) is not None - old_blocks = list(manager.get_blocks(request.request_id).blocks[0]) - command = coordinator.build_offload_command() - counts = {transfer.transfer_id: 2 for transfer in command.page_transfers} - partial = dict.fromkeys(counts, 1) - first_completions = partial.copy() - first_completions[next(iter(counts))] = 2 - coordinator.update_spills(counts, first_completions) - request.status = RequestStatus.FINISHED_LENGTH_CAPPED - manager.free(request) - replacement = make_request( - "reused", list(range(100, 117)), HISPARSE_BLOCK_SIZE, sha256 - ) - assert _allocate_scheduled(manager, replacement, 17) is not None - new_blocks = list(manager.get_blocks(replacement.request_id).blocks[0]) - pressure = host_pool.get_new_blocks(host_pool.get_num_free_blocks()) - assert not {block.block_id for block in old_blocks[:2]} & { - block.block_id for block in pressure + new_blocks - } - # Only the page every worker finished writing is published. - assert old_blocks[0].block_hash is not None - assert all(block.block_hash is None for block in old_blocks[1:]) - coordinator.update_spills({}, partial) - repeated = make_request("probe", tokens, HISPARSE_BLOCK_SIZE, sha256) - assert manager.get_computed_blocks(repeated)[1] == 2 * HISPARSE_BLOCK_SIZE - assert all(block.block_hash is None for block in new_blocks) - assert all(block.ref_cnt == 0 for block in old_blocks[:2]) - coordinator.update_spills({}, partial) - assert all(block.ref_cnt == 0 for block in old_blocks[:2]) - host_pool.free_blocks(pressure) - manager.free(replacement) - _publish_hisparse_pages(manager) - assert not coordinator.has_pending_work() - - -def test_hisparse_reset_prefix_cache_drops_copies(): - """Reset must forget GPU copies along with the host hashes they mirror.""" - manager = make_hisparse_kv_cache_manager(32, 16, enable_caching=True) - tokens = list(range(4 * HISPARSE_BLOCK_SIZE)) - request = make_request("request", tokens, HISPARSE_BLOCK_SIZE, sha256) - assert _allocate_scheduled(manager, request, num_new_tokens=len(tokens)) is not None - _publish_hisparse_pages(manager) - manager.free(request) - - coordinator = get_hisparse_coordinator(manager) - assert coordinator.copies - assert manager.reset_prefix_cache() - assert not coordinator.copies - - -def test_hisparse_reused_copy_is_not_adopted(): - """A host prefix hit must not adopt a GPU copy the pool has handed out.""" - manager = make_hisparse_kv_cache_manager(32, 16, enable_caching=True) - tokens = list(range(4 * HISPARSE_BLOCK_SIZE)) - original = make_request("original", tokens, HISPARSE_BLOCK_SIZE, sha256) - assert ( - _allocate_scheduled(manager, original, num_new_tokens=len(tokens)) is not None - ) - _publish_hisparse_pages(manager) - indexer_blocks = list(manager.get_blocks("original").blocks[1]) - manager.free(original) - pool = manager.block_pool - pool.touch(indexer_blocks) - reused = pool.get_new_blocks(pool.get_num_free_blocks()) - pool.free_blocks(reused) - pool.free_blocks(indexer_blocks) - - resumed = make_request("resumed", tokens, HISPARSE_BLOCK_SIZE, sha256) - computed, num_computed, _ = manager.get_computed_blocks(resumed) - assert num_computed == 3 * HISPARSE_BLOCK_SIZE - assert _allocate_scheduled( - manager, - resumed, - num_new_tokens=len(tokens) - num_computed, - num_new_computed_tokens=num_computed, - new_computed_blocks=computed, - ) - assert _resident_block_ids(manager, "resumed")[:3] == [0, 0, 0] - - -def test_hisparse_external_import_uses_hard_gpu_footprint(): - """An external prefix larger than resident capacity must remain admissible. - - This guards the admission boundary where indexer + one writable resident - page + the fixed hot allocation fit, but indexer + full resident history do - not. Requiring one more resident page would silently strand long P/D inputs. - """ - num_prompt_blocks = 4 - num_prompt_tokens = num_prompt_blocks * HISPARSE_BLOCK_SIZE - manager = make_hisparse_kv_cache_manager( - 8, - 9, - transfer_device_cache=True, - ) - request = make_request( - "host-import", - list(range(num_prompt_tokens)), - HISPARSE_BLOCK_SIZE, - sha256, - ) - - allocated = allocate_external_prefix(manager, request, num_prompt_tokens) - - assert allocated is not None - source, indexer, resident, hot = manager.get_blocks(request.request_id).blocks - assert len(source) == num_prompt_blocks - assert len(indexer) == num_prompt_blocks - assert len(resident) == num_prompt_blocks - assert all(block.is_null for block in resident[:-1]) - assert not resident[-1].is_null - assert len(hot) == 2 - assert manager.block_pool.get_num_free_blocks() == 0 - - -def test_hisparse_external_import_survives_capacity_retry(): - manager = make_hisparse_kv_cache_manager( - 9, - 7, - transfer_device_cache=True, - ) - tokens = list(range(4 * HISPARSE_BLOCK_SIZE)) - first = make_request("first", tokens, HISPARSE_BLOCK_SIZE, sha256) - second = make_request("second", tokens, HISPARSE_BLOCK_SIZE, sha256) - - assert allocate_external_prefix(manager, first, len(tokens)) is not None - - assert allocate_external_prefix(manager, second, len(tokens)) is None - - manager.free(first) - assert allocate_external_prefix(manager, second, len(tokens)) is not None - _, _, resident, hot = manager.get_blocks(second.request_id).blocks - assert all(block.is_null for block in resident[:-1]) - assert not resident[-1].is_null - assert len(hot) == 2 - - def make_kv_cache_config_hybrid_model( block_size: int, num_blocks: int, @@ -5404,31 +4031,6 @@ def test_can_fit_full_sequence_full_attention_still_gates_oversized(): assert manager.allocate_slots(req, block_size, full_sequence_must_fit=True) is None -def test_can_fit_full_sequence_hisparse_caps_resident_pages(): - manager = make_hisparse_kv_cache_manager( - num_blocks=11, - host_num_blocks=9, - max_model_len=8 * HISPARSE_BLOCK_SIZE, - max_in_flight_tokens=2 * HISPARSE_BLOCK_SIZE, - ) - request = make_request( - "long-hisparse", - list(range(8 * HISPARSE_BLOCK_SIZE)), - HISPARSE_BLOCK_SIZE, - sha256, - ) - - assert ( - _allocate_scheduled( - manager, - request, - num_new_tokens=HISPARSE_BLOCK_SIZE, - full_sequence_must_fit=True, - ) - is not None - ) - - def test_cache_hit_local_and_external(): # Regression test for #33775: when a request hits the local prefix cache # in one KV cache group and needs external (connector) blocks in another, @@ -6233,27 +4835,6 @@ def test_swa_shared_prefix_reuse_under_zero_retention(): assert last_req_hit(retention=0, pin=True) == 4 * block_size -def test_device_eviction_keeps_host_prefix(): - """Device eviction must not target a same-numbered host block.""" - manager = make_hisparse_kv_cache_manager(32, 16, enable_caching=True) - request = make_request("producer", list(range(64)), HISPARSE_BLOCK_SIZE, sha256) - assert _allocate_scheduled(manager, request, 64) is not None - _publish_hisparse_pages(manager) - host, indexer, _, _ = manager.get_blocks(request.request_id).blocks - device_block = indexer[0] - host_block = host[0] - assert device_block.block_id == host_block.block_id - manager.free(request) - assert host_block.block_hash is not None and device_block.block_hash is not None - manager.evict_blocks({device_block.block_id}) - assert device_block.block_hash is None, ( - "Device eviction left the indexer prefix cached" - ) - assert host_block.block_hash is not None, ( - "Device eviction invalidated unrelated host KV" - ) - - def test_get_unhashed_block_ids_all_groups(): """Unhashed, non-null block ids are reported per KV cache group. A group with no unhashed blocks reports an empty list, not a missing entry. diff --git a/tests/v1/core/test_scheduler.py b/tests/v1/core/test_scheduler.py index dcc0e23431..989c0e3c1a 100644 --- a/tests/v1/core/test_scheduler.py +++ b/tests/v1/core/test_scheduler.py @@ -24,12 +24,7 @@ from vllm.distributed.aux_output_connector.connector import ( AuxRequestOutput, ) from vllm.distributed.ec_transfer.ec_connector.metrics import ECConnectorStats -from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.connector import ( - HiSparseConnector, - HiSparseConnectorScheduler, -) from vllm.distributed.kv_transfer.kv_connector.v1.metrics import KVConnectorStats -from vllm.distributed.kv_transfer.kv_connector.v1.multi_connector import MultiConnector from vllm.multimodal.inputs import ( MultiModalFeatureSpec, MultiModalKwargsItem, @@ -49,7 +44,6 @@ from vllm.v1.core.single_type_kv_cache_manager import register_all_kvcache_specs from vllm.v1.engine import FinishReason from vllm.v1.engine.core import EngineCore from vllm.v1.executor.uniproc_executor import UniProcExecutor -from vllm.v1.hisparse.coordinator import get_hisparse_coordinator from vllm.v1.kv_cache_interface import ( FullAttentionSpec, KVCacheConfig, @@ -219,30 +213,6 @@ def test_max_num_active_seqs_caps_admission(): assert scheduler.max_num_running_reqs == 16 -def _bind_hisparse_connector(scheduler): - connector = object.__new__(HiSparseConnector) - connector.connector_scheduler = HiSparseConnectorScheduler(async_speculative=False) - from tests.v1.core.test_prefix_caching import make_hisparse_kv_cache_manager - - coordinator = get_hisparse_coordinator(make_hisparse_kv_cache_manager(16, 16)) - connector.connector_scheduler.bind_coordinator(coordinator) - composite = object.__new__(MultiConnector) - composite._connectors = (connector,) - scheduler.connector = composite - return coordinator - - -def test_pending_hisparse_spill_keeps_scheduler_alive(): - """A final host spill must complete after the last request finishes.""" - scheduler = create_scheduler() - coordinator = _bind_hisparse_connector(scheduler) - pending = coordinator.pending_spills - pending[0] = Mock() - assert scheduler.has_requests() - pending.clear() - assert not scheduler.has_requests() - - @pytest.mark.parametrize( "enable_prefix_caching, prompt_logprobs", [ @@ -1776,45 +1746,6 @@ def test_preempt_during_execution(): assert requests[1].output_token_ids[0] == 42 -def test_pending_hisparse_reclamation_defers_preemption(monkeypatch): - """Pending reclamation must stall rather than preempt its owning request.""" - scheduler = create_scheduler(enable_prefix_caching=False) - request = create_requests(num_requests=1, num_tokens=16, block_size=16)[0] - scheduler.add_request(request) - scheduler_output = scheduler.schedule() - scheduler.update_from_output( - scheduler_output, - ModelRunnerOutput( - req_ids=[request.request_id], - req_id_to_index={request.request_id: 0}, - sampled_token_ids=[[0]], - logprobs=None, - prompt_logprobs_dict={}, - pooler_output=[], - ), - ) - - monkeypatch.setattr( - scheduler.kv_cache_manager, "allocate_slots", Mock(return_value=None) - ) - coordinator = _bind_hisparse_connector(scheduler) - monkeypatch.setattr( - scheduler.connector, "build_connector_meta", Mock(return_value=None) - ) - monkeypatch.setattr( - coordinator, - "has_pending_reclamation", - Mock(return_value=True), - ) - - deferred_output = scheduler.schedule() - - assert not deferred_output.num_scheduled_tokens - assert scheduler.running == [request] - assert request.status == RequestStatus.RUNNING - assert request.num_preemptions == 0 - - def test_prefix_cache_query_not_inflated_by_connector_defer(): """The GPU prefix-cache query is recorded at admission, so a request the connector defers several times is counted once, not once per retry.""" diff --git a/tests/v1/kv_connector/unit/test_bidirectional_kv_transfer.py b/tests/v1/kv_connector/unit/test_bidirectional_kv_transfer.py deleted file mode 100644 index 2aae7d71f5..0000000000 --- a/tests/v1/kv_connector/unit/test_bidirectional_kv_transfer.py +++ /dev/null @@ -1,1082 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Unit tests for bi-directional KV cache transfer between P and D nodes. - -Tests cover the new behaviors added by the bi-directional KV transfer PR: -1. P-node scheduler lifecycle: P pulls KV from D using remote_block_ids, - eliminating redundant prefill computation in multi-turn conversations. -2. P-node metadata: NixlConnectorMetadata correctly populates recv metadata - when P pulls KV from D (do_remote_decode=True + remote_block_ids). -3. P-node worker: start_load_kv processes reqs_to_recv for KV pull from D. -4. D-node request_finished: returns kv_transfer_params with remote_block_ids - and remote_num_tokens so P can pull KV in future turns. -5. Edge cases: - - No double read after reschedule (_remote_blocks_processed flag) - - remote_num_tokens bounded by block capacity (num_computed_tokens) - - kv_recompute_threshold skips small transfers - - P-node holds blocks for D after finishing - - Cache MISS first turn falls back to local prefill - - Partial remote coverage: P pulls partial, computes the rest - - _remote_blocks_processed flag persists across reschedules - -P-node flags: do_remote_prefill=False (prefill locally), -do_remote_decode=True (don't decode locally, send KV to D). -P pulls KV from D when remote_block_ids is not None and -external tokens > 0. -""" - -import copy -import time -from unittest.mock import patch - -import pytest - -from vllm.distributed.kv_transfer.kv_connector.v1.base import KVConnectorRole -from vllm.distributed.kv_transfer.kv_connector.v1.nixl import ( - NixlConnector, - NixlConnectorMetadata, -) -from vllm.forward_context import ForwardContext -from vllm.v1.outputs import ( - EMPTY_MODEL_RUNNER_OUTPUT, - KVConnectorOutput, -) -from vllm.v1.request import RequestStatus - -from .test_nixl_connector import FakeNixlConnectorWorker, FakeNixlWrapper -from .utils import ( - assert_scheduler_empty, - create_model_runner_output, - create_request, - create_scheduler, - create_vllm_config, - make_kv_cache_config, - maybe_update_block_size, -) - -pytestmark = pytest.mark.cpu_test - -# Common extra config for all bi-directional KV transfer tests. -BIDIR_KV_EXTRA_CONFIG = {"bidirectional_kv_xfer": True, "kv_recompute_threshold": 0} - - -# Helpers - - -def _make_p_node_turn2_request( - request_id, block_size, num_tokens, num_remote_blocks=3, remote_num_tokens=None -): - """Create a P-node Turn 2 request with remote_block_ids from D.""" - request = create_request( - request_id=request_id, - block_size=block_size, - num_tokens=num_tokens, - do_remote_decode=True, - ) - if remote_num_tokens is None: - remote_num_tokens = num_remote_blocks * block_size - request.kv_transfer_params["remote_block_ids"] = [list(range(num_remote_blocks))] - request.kv_transfer_params["remote_num_tokens"] = remote_num_tokens - request.kv_transfer_params["remote_engine_id"] = "decode-engine" - request.kv_transfer_params["remote_request_id"] = f"decode-{request_id}" - request.kv_transfer_params["remote_host"] = "decode-host" - request.kv_transfer_params["remote_port"] = 5678 - return request - - -def _make_connector_with_fake_worker( - hand_shake_latency=0, cycles_before_done=0, do_handshake=True -): - """Create a NixlConnector with FakeNixlConnectorWorker.""" - block_size = maybe_update_block_size(32) - vllm_config = create_vllm_config( - kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG, block_size=block_size - ) - kv_cache_config = make_kv_cache_config(block_size=block_size, num_blocks=2) - connector = NixlConnector(vllm_config, KVConnectorRole.WORKER, kv_cache_config) - connector.connector_worker = FakeNixlConnectorWorker( - vllm_config, - connector.engine_id, - hand_shake_latency=hand_shake_latency, - kv_cache_config=kv_cache_config, - ) - worker = connector.connector_worker - assert isinstance(worker.nixl_wrapper, FakeNixlWrapper) - worker.kv_cache_layout = "LBHNC" - if do_handshake: - remote_agents, _ = worker._nixl_handshake( - host="localhost", - port=1234, - remote_tp_size=1, - expected_engine_id=FakeNixlConnectorWorker.REMOTE_ENGINE_ID, - ) - worker._remote_agents[FakeNixlConnectorWorker.REMOTE_ENGINE_ID] = remote_agents - return connector, worker - - -def _make_p_node_recv_metadata(request_id, local_blocks, remote_blocks): - """Build NixlConnectorMetadata for P-node pulling KV from D.""" - meta = NixlConnectorMetadata() - meta.add_new_req_to_recv( - request_id=request_id, - local_block_ids=(local_blocks,), - kv_transfer_params={ - "do_remote_prefill": False, - "do_remote_decode": True, - "remote_block_ids": (remote_blocks,), - "remote_engine_id": FakeNixlConnectorWorker.REMOTE_ENGINE_ID, - "remote_request_id": f"decode-{request_id}", - "remote_host": "localhost", - "remote_port": 1234, - "remote_tp_size": 1, - }, - ) - return meta - - -def _do_load_kv(connector, metadata): - """Bind metadata and call start_load_kv.""" - connector.bind_connector_metadata(metadata) - ctx = ForwardContext(no_compile_layers={}, attn_metadata={}, slot_mapping={}) - connector.start_load_kv(ctx) - - -# 1. P-node scheduler lifecycle tests - - -def test_multiturn_lifecycle(): - """Full two-turn lifecycle on the P node: - Turn 1: P prefills locally (do_remote_prefill=False), sends KV to D - (do_remote_decode=True). Finishes LENGTH_CAPPED with remote_block_ids. - Turn 2: P receives remote_block_ids from D. P pulls KV from D because - remote_block_ids is not None and external tokens > 0. Computes only - new tokens, finishes LENGTH_CAPPED.""" - vllm_config = create_vllm_config( - kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG, - ) - scheduler = create_scheduler(vllm_config) - BS = vllm_config.cache_config.block_size - - t1 = create_request( - request_id=100, block_size=BS, num_tokens=int(BS * 2.5), do_remote_decode=True - ) - scheduler.add_request(t1) - t1_id = t1.request_id - so = scheduler.schedule() - mro = create_model_runner_output(reqs=[t1]) - eco = scheduler.update_from_output(so, mro) - assert t1.status == RequestStatus.FINISHED_LENGTH_CAPPED - kv = eco[0].outputs[0].kv_transfer_params - assert kv and sum(len(g) for g in kv["remote_block_ids"]) > 0 - so = scheduler.schedule() - scheduler.update_from_output(so, EMPTY_MODEL_RUNNER_OUTPUT) - - t2 = _make_p_node_turn2_request(200, BS, int(BS * 2.5)) - scheduler.add_request(t2) - t2_id = t2.request_id - so = scheduler.schedule() - assert t2.status == RequestStatus.WAITING_FOR_REMOTE_KVS - scheduler.update_from_output(so, EMPTY_MODEL_RUNNER_OUTPUT) - so = scheduler.schedule() - mro = copy.deepcopy(EMPTY_MODEL_RUNNER_OUTPUT) - mro.kv_connector_output = KVConnectorOutput(finished_recving={t2_id}) - scheduler.update_from_output(so, mro) - so = scheduler.schedule() - mro = create_model_runner_output(reqs=[t2]) - scheduler.update_from_output(so, mro) - assert t2.status == RequestStatus.FINISHED_LENGTH_CAPPED - so = scheduler.schedule() - scheduler.update_from_output(so, EMPTY_MODEL_RUNNER_OUTPUT) - so = scheduler.schedule() - mro = copy.deepcopy(EMPTY_MODEL_RUNNER_OUTPUT) - mro.kv_connector_output = KVConnectorOutput(finished_sending={t1_id, t2_id}) - scheduler.update_from_output(so, mro) - assert_scheduler_empty(scheduler) - - -def test_first_turn_no_remote_blocks(): - """First turn: P has no remote_block_ids from D yet. - Standard local prefill, returns kv_transfer_params for future turns.""" - vllm_config = create_vllm_config( - kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG, - ) - scheduler = create_scheduler(vllm_config) - BS = vllm_config.cache_config.block_size - req = create_request( - request_id=3, block_size=BS, num_tokens=int(BS * 2.5), do_remote_decode=True - ) - scheduler.add_request(req) - req_id = req.request_id - so = scheduler.schedule() - assert req.status != RequestStatus.WAITING_FOR_REMOTE_KVS - mro = create_model_runner_output(reqs=[req]) - eco = scheduler.update_from_output(so, mro) - assert req.status == RequestStatus.FINISHED_LENGTH_CAPPED - assert eco[0].outputs[0].kv_transfer_params is not None - so = scheduler.schedule() - scheduler.update_from_output(so, EMPTY_MODEL_RUNNER_OUTPUT) - so = scheduler.schedule() - mro = copy.deepcopy(EMPTY_MODEL_RUNNER_OUTPUT) - mro.kv_connector_output = KVConnectorOutput(finished_sending={req_id}) - scheduler.update_from_output(so, mro) - assert_scheduler_empty(scheduler) - - -def test_abort_p_side_during_send(): - """P-side do_remote_decode=True: blocks held until finished_sending.""" - vllm_config = create_vllm_config( - kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG, - ) - scheduler = create_scheduler(vllm_config) - BS = vllm_config.cache_config.block_size - req = create_request( - request_id=42, block_size=BS, num_tokens=int(BS * 2.5), do_remote_decode=True - ) - scheduler.add_request(req) - req_id = req.request_id - so = scheduler.schedule() - mro = create_model_runner_output(reqs=[req]) - scheduler.update_from_output(so, mro) - assert req_id in scheduler.requests - so = scheduler.schedule() - scheduler.update_from_output(so, EMPTY_MODEL_RUNNER_OUTPUT) - assert req_id in scheduler.requests - so = scheduler.schedule() - mro = copy.deepcopy(EMPTY_MODEL_RUNNER_OUTPUT) - mro.kv_connector_output = KVConnectorOutput(finished_sending={req_id}) - scheduler.update_from_output(so, mro) - assert_scheduler_empty(scheduler) - - -def test_abort_p_side_non_length_capped(): - """P-side abort with non-LENGTH_CAPPED → immediate block free.""" - vllm_config = create_vllm_config( - kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG, - ) - scheduler = create_scheduler(vllm_config) - BS = vllm_config.cache_config.block_size - req = create_request( - request_id=44, block_size=BS, num_tokens=int(BS * 2.5), do_remote_decode=True - ) - req.sampling_params.max_tokens = 100 - req.max_tokens = 100 - scheduler.add_request(req) - req_id = req.request_id - so = scheduler.schedule() - mro = create_model_runner_output(reqs=[req]) - scheduler.update_from_output(so, mro) - scheduler.finish_requests([req_id], RequestStatus.FINISHED_ABORTED) - conn = scheduler.connector.connector_scheduler - assert req_id in conn._reqs_not_processed - assert req_id not in scheduler.requests - so = scheduler.schedule() - scheduler.update_from_output(so, EMPTY_MODEL_RUNNER_OUTPUT) - assert_scheduler_empty(scheduler) - - -def test_remote_blocks_exceed_prompt_tokens(): - """D provides more remote tokens than P's prompt needs. - P caps external tokens to prompt length.""" - vllm_config = create_vllm_config( - kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG, - ) - scheduler = create_scheduler(vllm_config) - BS = vllm_config.cache_config.block_size - NUM_TOKENS = int(BS * 2.5) - req = _make_p_node_turn2_request( - 300, BS, NUM_TOKENS, num_remote_blocks=5, remote_num_tokens=5 * BS - ) - scheduler.add_request(req) - req_id = req.request_id - so = scheduler.schedule() - assert req.status == RequestStatus.WAITING_FOR_REMOTE_KVS - assert req.num_computed_tokens == NUM_TOKENS - scheduler.update_from_output(so, EMPTY_MODEL_RUNNER_OUTPUT) - so = scheduler.schedule() - mro = copy.deepcopy(EMPTY_MODEL_RUNNER_OUTPUT) - mro.kv_connector_output = KVConnectorOutput(finished_recving={req_id}) - scheduler.update_from_output(so, mro) - so = scheduler.schedule() - mro = create_model_runner_output(reqs=[req]) - scheduler.update_from_output(so, mro) - assert req.status == RequestStatus.FINISHED_LENGTH_CAPPED - so = scheduler.schedule() - scheduler.update_from_output(so, EMPTY_MODEL_RUNNER_OUTPUT) - so = scheduler.schedule() - mro = copy.deepcopy(EMPTY_MODEL_RUNNER_OUTPUT) - mro.kv_connector_output = KVConnectorOutput(finished_sending={req_id}) - scheduler.update_from_output(so, mro) - assert_scheduler_empty(scheduler) - - -def test_p_node_pulls_partial_last_block_from_d(): - """D sends remote_block_ids with partially filled last block. - remote_num_tokens < len(remote_block_ids) * block_size. - P pulls only remote_num_tokens worth of external tokens.""" - vllm_config = create_vllm_config( - kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG, - ) - scheduler = create_scheduler(vllm_config) - BS = vllm_config.cache_config.block_size - num_remote_blocks = 3 - remote_num_tokens = int(BS * 2.5) - assert remote_num_tokens < num_remote_blocks * BS - NUM_TOKENS = int(BS * 3.5) - req = _make_p_node_turn2_request( - 400, - BS, - NUM_TOKENS, - num_remote_blocks=num_remote_blocks, - remote_num_tokens=remote_num_tokens, - ) - scheduler.add_request(req) - req_id = req.request_id - so = scheduler.schedule() - assert req.status == RequestStatus.WAITING_FOR_REMOTE_KVS - scheduler.update_from_output(so, EMPTY_MODEL_RUNNER_OUTPUT) - so = scheduler.schedule() - mro = copy.deepcopy(EMPTY_MODEL_RUNNER_OUTPUT) - mro.kv_connector_output = KVConnectorOutput(finished_recving={req_id}) - scheduler.update_from_output(so, mro) - so = scheduler.schedule() - assert len(scheduler.running) == 1 - mro = create_model_runner_output(reqs=[req]) - scheduler.update_from_output(so, mro) - assert req.status == RequestStatus.FINISHED_LENGTH_CAPPED - so = scheduler.schedule() - scheduler.update_from_output(so, EMPTY_MODEL_RUNNER_OUTPUT) - so = scheduler.schedule() - mro = copy.deepcopy(EMPTY_MODEL_RUNNER_OUTPUT) - mro.kv_connector_output = KVConnectorOutput(finished_sending={req_id}) - scheduler.update_from_output(so, mro) - assert_scheduler_empty(scheduler) - - -# 2. P-node metadata tests - - -def test_add_new_req_to_recv_populates_remote_meta(): - """add_new_req_to_recv correctly populates RemoteMeta for P-node - bi-directional KV pull from D.""" - meta = NixlConnectorMetadata() - kv_params = { - "remote_block_ids": [[0, 1, 2]], - "remote_engine_id": "decode-engine", - "remote_request_id": "decode-req-123", - "remote_host": "decode-host", - "remote_port": 5678, - } - local_block_ids = ([10, 11, 12],) - meta.add_new_req_to_recv( - request_id="test-req", - local_block_ids=local_block_ids, - kv_transfer_params=kv_params, - ) - assert "test-req" in meta.reqs_to_recv - rm = meta.reqs_to_recv["test-req"] - assert rm.remote is not None - assert rm.remote.block_ids == kv_params["remote_block_ids"] - assert rm.remote.engine_id == "decode-engine" - assert rm.remote.request_id == "decode-req-123" - assert rm.remote.host == "decode-host" - assert rm.remote.port == 5678 - assert rm.local_block_ids == local_block_ids - - -def test_build_connector_meta_recv_entries(): - """P-node scheduler: do_remote_decode=True + remote_block_ids → - _reqs_need_recv populated, build_connector_meta produces reqs_to_recv.""" - vllm_config = create_vllm_config( - kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG, - ) - scheduler = create_scheduler(vllm_config) - BS = vllm_config.cache_config.block_size - req = _make_p_node_turn2_request(1, BS, int(BS * 2.5)) - scheduler.add_request(req) - req_id = req.request_id - so = scheduler.schedule() - assert req.status == RequestStatus.WAITING_FOR_REMOTE_KVS - meta = so.kv_connector_metadata - assert isinstance(meta, NixlConnectorMetadata) - assert req_id in meta.reqs_to_recv - rm = meta.reqs_to_recv[req_id] - assert rm.remote is not None - assert rm.remote.engine_id == "decode-engine" - - -def test_build_connector_meta_clears_reqs_need_recv(): - """After build_connector_meta, _reqs_need_recv is cleared.""" - vllm_config = create_vllm_config( - kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG, - ) - scheduler = create_scheduler(vllm_config) - BS = vllm_config.cache_config.block_size - req = _make_p_node_turn2_request(2, BS, int(BS * 2.5)) - scheduler.add_request(req) - conn = scheduler.connector.connector_scheduler - scheduler.schedule() - assert len(conn._reqs_need_recv) == 0 - - -def test_build_connector_meta_multiple_requests(): - """Multiple P-node requests all included in reqs_to_recv.""" - vllm_config = create_vllm_config( - kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG, - ) - scheduler = create_scheduler(vllm_config) - BS = vllm_config.cache_config.block_size - reqs = [_make_p_node_turn2_request(10 + i, BS, int(BS * 2.5)) for i in range(3)] - for r in reqs: - scheduler.add_request(r) - so = scheduler.schedule() - meta = so.kv_connector_metadata - assert isinstance(meta, NixlConnectorMetadata) - assert len(meta.reqs_to_recv) == 3 - for r in reqs: - assert r.request_id in meta.reqs_to_recv - - -# 3. P-node worker tests (FakeNixlWrapper) - - -@patch( - "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper", - FakeNixlWrapper, -) -def test_p_node_pull_kv_from_d(dist_init): - """P node pulls KV from D via start_load_kv with reqs_to_recv.""" - connector, worker = _make_connector_with_fake_worker() - meta = _make_p_node_recv_metadata("req-p1", [10, 11, 12], [20, 21, 22]) - _do_load_kv(connector, meta) - assert "req-p1" in worker._recving_metadata - _, done_recving = connector.get_finished(finished_req_ids=set()) - assert "req-p1" in done_recving - - -@patch( - "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper", - FakeNixlWrapper, -) -def test_p_node_pull_then_send_kv(dist_init): - """Full P-node bi-directional: pull KV from D → prefill → - send KV back to D via notification.""" - connector, worker = _make_connector_with_fake_worker() - meta = _make_p_node_recv_metadata("req-p2", [10, 11], [20, 21]) - _do_load_kv(connector, meta) - _, done_recving = connector.get_finished(finished_req_ids=set()) - assert "req-p2" in done_recving - worker._reqs_to_send["req-p2"] = time.perf_counter() + 60 - worker._reqs_to_process.add("req-p2") - notif = f"req-p2:{worker.world_size}".encode() - orig = worker.nixl_wrapper.get_new_notifs - worker.nixl_wrapper.get_new_notifs = lambda: {"agent": [notif]} - done_sending, _ = connector.get_finished(finished_req_ids=set()) - assert "req-p2" in done_sending - worker.nixl_wrapper.get_new_notifs = orig - - -@patch( - "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper", - FakeNixlWrapper, -) -def test_p_node_deferred_pull_on_no_handshake(dist_init): - """P defers KV pull when no prior handshake exists.""" - connector, worker = _make_connector_with_fake_worker( - hand_shake_latency=0, do_handshake=False - ) - meta = _make_p_node_recv_metadata("req-p3", [10, 11], [20, 21]) - _do_load_kv(connector, meta) - assert "req-p3" in worker._recving_metadata - timeout = 3.0 - start = time.perf_counter() - while time.perf_counter() - start < timeout: - connector.bind_connector_metadata(NixlConnectorMetadata()) - ctx = ForwardContext(no_compile_layers={}, attn_metadata={}, slot_mapping={}) - connector.start_load_kv(ctx) - _, done = connector.get_finished(finished_req_ids=set()) - if "req-p3" in done: - return - time.sleep(0.2) - raise AssertionError("Transfer did not complete after async handshake") - - -# 4. D-node request_finished returns kv_transfer_params (new behavior) - - -def test_d_node_request_finished_returns_kv_params(): - """D-node request_finished returns kv_transfer_params with - do_remote_decode=True, remote_block_ids, remote_num_tokens - for P to pull. These params go directly to P node.""" - vllm_config = create_vllm_config( - kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG, - ) - scheduler = create_scheduler(vllm_config) - BS = vllm_config.cache_config.block_size - req = create_request( - request_id=1, block_size=BS, num_tokens=int(BS * 2.5), do_remote_prefill=True - ) - scheduler.add_request(req) - req_id = req.request_id - so = scheduler.schedule() - scheduler.update_from_output( - so, create_model_runner_output(reqs=[], finished_recving={req_id}) - ) - so = scheduler.schedule() - eco = scheduler.update_from_output( - so, create_model_runner_output(reqs=[req], use_eos=True) - ) - assert req.status == RequestStatus.FINISHED_STOPPED - kv = eco[0].outputs[0].kv_transfer_params - assert kv is not None - assert kv["do_remote_decode"] is True - assert kv["do_remote_prefill"] is False - assert "remote_block_ids" in kv - assert "remote_num_tokens" in kv - assert kv["remote_num_tokens"] > 0 - - -def test_d_node_request_finished_delays_block_free(): - """D-node holds blocks (delay_free=True) until P reads them.""" - vllm_config = create_vllm_config( - kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG, - ) - scheduler = create_scheduler(vllm_config) - BS = vllm_config.cache_config.block_size - req = create_request( - request_id=2, block_size=BS, num_tokens=int(BS * 2.5), do_remote_prefill=True - ) - scheduler.add_request(req) - req_id = req.request_id - so = scheduler.schedule() - scheduler.update_from_output( - so, create_model_runner_output(reqs=[], finished_recving={req_id}) - ) - so = scheduler.schedule() - scheduler.update_from_output( - so, create_model_runner_output(reqs=[req], use_eos=True) - ) - assert req_id in scheduler.requests - conn = scheduler.connector.connector_scheduler - assert req_id in conn._reqs_need_send - - -def test_d_node_request_finished_remote_num_tokens(): - """D-node kv_transfer_params includes correct remote_num_tokens.""" - vllm_config = create_vllm_config( - kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG, - ) - scheduler = create_scheduler(vllm_config) - BS = vllm_config.cache_config.block_size - req = create_request( - request_id=3, block_size=BS, num_tokens=int(BS * 2.5), do_remote_prefill=True - ) - scheduler.add_request(req) - req_id = req.request_id - so = scheduler.schedule() - scheduler.update_from_output( - so, create_model_runner_output(reqs=[], finished_recving={req_id}) - ) - so = scheduler.schedule() - eco = scheduler.update_from_output( - so, create_model_runner_output(reqs=[req], use_eos=True) - ) - kv = eco[0].outputs[0].kv_transfer_params - assert kv["remote_num_tokens"] > 0 - assert sum(len(g) for g in kv["remote_block_ids"]) > 0 - - -def test_d_node_partial_last_block_remote_num_tokens(): - """D-node: remote_num_tokens < len(remote_block_ids) * block_size - when last block is partially filled.""" - vllm_config = create_vllm_config( - kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG, - ) - scheduler = create_scheduler(vllm_config) - BS = vllm_config.cache_config.block_size - req = create_request( - request_id=5, block_size=BS, num_tokens=int(BS * 2.5), do_remote_prefill=True - ) - scheduler.add_request(req) - req_id = req.request_id - so = scheduler.schedule() - scheduler.update_from_output( - so, create_model_runner_output(reqs=[], finished_recving={req_id}) - ) - so = scheduler.schedule() - eco = scheduler.update_from_output( - so, create_model_runner_output(reqs=[req], use_eos=True) - ) - kv = eco[0].outputs[0].kv_transfer_params - total_blocks = sum(len(g) for g in kv["remote_block_ids"]) - assert total_blocks == 3 - assert kv["remote_num_tokens"] < total_blocks * BS - assert kv["remote_num_tokens"] > 0 - - -# 5. Edge case tests - - -def test_no_double_read_blocks_after_reschedule(): - """Edge case 1: update_state_after_alloc called twice for the same - bidirectional request (once on initial schedule, once after - WAITING_FOR_REMOTE_KVS → reschedule). The _remote_blocks_processed - flag must prevent the request from being added to _reqs_need_recv - twice, which would cause P to read D's blocks twice.""" - vllm_config = create_vllm_config( - kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG, - ) - scheduler = create_scheduler(vllm_config) - BS = vllm_config.cache_config.block_size - req = _make_p_node_turn2_request(500, BS, int(BS * 2.5)) - scheduler.add_request(req) - req_id = req.request_id - conn = scheduler.connector.connector_scheduler - - # First schedule: request enters WAITING_FOR_REMOTE_KVS, - # _reqs_need_recv populated then cleared by build_connector_meta. - so = scheduler.schedule() - assert req.status == RequestStatus.WAITING_FOR_REMOTE_KVS - meta = so.kv_connector_metadata - assert isinstance(meta, NixlConnectorMetadata) - assert req_id in meta.reqs_to_recv - # _reqs_need_recv should be cleared after build_connector_meta - assert len(conn._reqs_need_recv) == 0 - - # Simulate recv completion - scheduler.update_from_output(so, EMPTY_MODEL_RUNNER_OUTPUT) - so = scheduler.schedule() - mro = copy.deepcopy(EMPTY_MODEL_RUNNER_OUTPUT) - mro.kv_connector_output = KVConnectorOutput(finished_recving={req_id}) - scheduler.update_from_output(so, mro) - - # Second schedule after recv: update_state_after_alloc called again. - # The _remote_blocks_processed flag should prevent re-entry. - so = scheduler.schedule() - meta2 = so.kv_connector_metadata - assert isinstance(meta2, NixlConnectorMetadata) - # Must NOT be in reqs_to_recv again - assert req_id not in meta2.reqs_to_recv - - # Clean up - mro = create_model_runner_output(reqs=[req]) - scheduler.update_from_output(so, mro) - assert req.status == RequestStatus.FINISHED_LENGTH_CAPPED - so = scheduler.schedule() - scheduler.update_from_output(so, EMPTY_MODEL_RUNNER_OUTPUT) - so = scheduler.schedule() - mro = copy.deepcopy(EMPTY_MODEL_RUNNER_OUTPUT) - mro.kv_connector_output = KVConnectorOutput(finished_sending={req_id}) - scheduler.update_from_output(so, mro) - assert_scheduler_empty(scheduler) - - -def test_remote_num_tokens_bounded_by_blocks(): - """Edge case 2: D-node request_finished must return - remote_num_tokens <= len(remote_block_ids) * block_size. - request.num_tokens includes the last sampled token which has no KV - in the cache, so remote_num_tokens must use num_computed_tokens.""" - vllm_config = create_vllm_config( - kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG, - ) - scheduler = create_scheduler(vllm_config) - BS = vllm_config.cache_config.block_size - req = create_request( - request_id=501, - block_size=BS, - num_tokens=int(BS * 2.5), - do_remote_prefill=True, - ) - scheduler.add_request(req) - req_id = req.request_id - so = scheduler.schedule() - scheduler.update_from_output( - so, create_model_runner_output(reqs=[], finished_recving={req_id}) - ) - so = scheduler.schedule() - eco = scheduler.update_from_output( - so, create_model_runner_output(reqs=[req], use_eos=True) - ) - kv = eco[0].outputs[0].kv_transfer_params - assert kv is not None - total_blocks = sum(len(g) for g in kv["remote_block_ids"]) - max_tokens_in_blocks = total_blocks * BS - assert kv["remote_num_tokens"] <= max_tokens_in_blocks, ( - f"remote_num_tokens ({kv['remote_num_tokens']}) exceeds " - f"block capacity ({max_tokens_in_blocks})" - ) - assert kv["remote_num_tokens"] > 0 - - -def test_kv_recompute_threshold_skips_small_transfer(): - """Edge case 3: When remote tokens are below kv_recompute_threshold, - P should skip the remote pull and compute locally instead of - entering WAITING_FOR_REMOTE_KVS.""" - threshold = 256 - vllm_config = create_vllm_config( - kv_connector_extra_config={ - "bidirectional_kv_xfer": True, - "kv_recompute_threshold": threshold, - }, - ) - scheduler = create_scheduler(vllm_config) - BS = vllm_config.cache_config.block_size - - # Create request where remote tokens (48) < threshold (256) - req = _make_p_node_turn2_request( - 502, - BS, - int(BS * 2.5), - num_remote_blocks=3, - remote_num_tokens=3 * BS, - ) - scheduler.add_request(req) - so = scheduler.schedule() - # Should NOT enter WAITING_FOR_REMOTE_KVS — threshold not met - assert req.status != RequestStatus.WAITING_FOR_REMOTE_KVS - assert req.status == RequestStatus.RUNNING - - # Clean up - mro = create_model_runner_output(reqs=[req]) - scheduler.update_from_output(so, mro) - assert req.status == RequestStatus.FINISHED_LENGTH_CAPPED - so = scheduler.schedule() - scheduler.update_from_output(so, EMPTY_MODEL_RUNNER_OUTPUT) - so = scheduler.schedule() - mro = copy.deepcopy(EMPTY_MODEL_RUNNER_OUTPUT) - mro.kv_connector_output = KVConnectorOutput(finished_sending={req.request_id}) - scheduler.update_from_output(so, mro) - assert_scheduler_empty(scheduler) - - -def test_p_node_finished_holds_blocks_for_d(): - """Edge case 4: P-node finishes with FINISHED_LENGTH_CAPPED and - do_remote_decode=True. P must hold blocks (delay_free=True) and - return kv_transfer_params with do_remote_prefill=True so D can - read P's blocks.""" - vllm_config = create_vllm_config( - kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG, - ) - scheduler = create_scheduler(vllm_config) - BS = vllm_config.cache_config.block_size - req = create_request( - request_id=503, - block_size=BS, - num_tokens=int(BS * 2.5), - do_remote_decode=True, - ) - scheduler.add_request(req) - req_id = req.request_id - so = scheduler.schedule() - mro = create_model_runner_output(reqs=[req]) - eco = scheduler.update_from_output(so, mro) - assert req.status == RequestStatus.FINISHED_LENGTH_CAPPED - kv = eco[0].outputs[0].kv_transfer_params - assert kv is not None - # P-node finished: should tell D to pull (do_remote_prefill=True) - assert kv["do_remote_prefill"] is True - assert kv["do_remote_decode"] is False - assert "remote_block_ids" in kv - assert sum(len(g) for g in kv["remote_block_ids"]) > 0 - # Blocks should be held (request still tracked) - assert req_id in scheduler.requests - - # Clean up: simulate D reading and notifying - so = scheduler.schedule() - scheduler.update_from_output(so, EMPTY_MODEL_RUNNER_OUTPUT) - so = scheduler.schedule() - mro = copy.deepcopy(EMPTY_MODEL_RUNNER_OUTPUT) - mro.kv_connector_output = KVConnectorOutput(finished_sending={req_id}) - scheduler.update_from_output(so, mro) - assert_scheduler_empty(scheduler) - - -def test_cache_miss_first_turn_no_remote_pull(): - """Edge case 5: First turn with do_remote_decode=True but no - remote_block_ids (cache MISS). P should prefill locally with - num_external_tokens=0 and not enter WAITING_FOR_REMOTE_KVS.""" - vllm_config = create_vllm_config( - kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG, - ) - scheduler = create_scheduler(vllm_config) - BS = vllm_config.cache_config.block_size - req = create_request( - request_id=504, - block_size=BS, - num_tokens=int(BS * 2.5), - do_remote_decode=True, - ) - # No remote_block_ids set — this is a cache MISS - assert req.kv_transfer_params.get("remote_block_ids") is None - scheduler.add_request(req) - so = scheduler.schedule() - # Should NOT wait for remote KVs - assert req.status != RequestStatus.WAITING_FOR_REMOTE_KVS - assert req.status == RequestStatus.RUNNING - - # Clean up - mro = create_model_runner_output(reqs=[req]) - scheduler.update_from_output(so, mro) - so = scheduler.schedule() - scheduler.update_from_output(so, EMPTY_MODEL_RUNNER_OUTPUT) - so = scheduler.schedule() - mro = copy.deepcopy(EMPTY_MODEL_RUNNER_OUTPUT) - mro.kv_connector_output = KVConnectorOutput(finished_sending={req.request_id}) - scheduler.update_from_output(so, mro) - assert_scheduler_empty(scheduler) - - -def test_partial_remote_tokens_less_than_prompt(): - """Edge case 6: D's remote_num_tokens covers only part of P's - prompt. P should pull remote_num_tokens worth of external tokens - and compute the rest locally.""" - vllm_config = create_vllm_config( - kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG, - ) - scheduler = create_scheduler(vllm_config) - BS = vllm_config.cache_config.block_size - NUM_TOKENS = int(BS * 4.5) # 72 tokens - # D provides only 2 blocks (32 tokens) out of 72 - req = _make_p_node_turn2_request( - 505, - BS, - NUM_TOKENS, - num_remote_blocks=2, - remote_num_tokens=2 * BS, - ) - scheduler.add_request(req) - req_id = req.request_id - so = scheduler.schedule() - assert req.status == RequestStatus.WAITING_FOR_REMOTE_KVS - # num_computed_tokens should reflect the external tokens pulled - # (capped to remote_num_tokens, not full prompt) - assert req.num_computed_tokens < NUM_TOKENS - - # Complete the transfer and finish - scheduler.update_from_output(so, EMPTY_MODEL_RUNNER_OUTPUT) - so = scheduler.schedule() - mro = copy.deepcopy(EMPTY_MODEL_RUNNER_OUTPUT) - mro.kv_connector_output = KVConnectorOutput(finished_recving={req_id}) - scheduler.update_from_output(so, mro) - so = scheduler.schedule() - mro = create_model_runner_output(reqs=[req]) - scheduler.update_from_output(so, mro) - assert req.status == RequestStatus.FINISHED_LENGTH_CAPPED - so = scheduler.schedule() - scheduler.update_from_output(so, EMPTY_MODEL_RUNNER_OUTPUT) - so = scheduler.schedule() - mro = copy.deepcopy(EMPTY_MODEL_RUNNER_OUTPUT) - mro.kv_connector_output = KVConnectorOutput(finished_sending={req_id}) - scheduler.update_from_output(so, mro) - assert_scheduler_empty(scheduler) - - -def test_remote_blocks_processed_flag_persists(): - """Edge case 7: After recv completes and request is rescheduled, - the _remote_blocks_processed flag in kv_transfer_params prevents - the bidirectional path from re-entering _reqs_need_recv.""" - vllm_config = create_vllm_config( - kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG, - ) - scheduler = create_scheduler(vllm_config) - BS = vllm_config.cache_config.block_size - req = _make_p_node_turn2_request(506, BS, int(BS * 2.5)) - scheduler.add_request(req) - req_id = req.request_id - conn = scheduler.connector.connector_scheduler - - # First schedule → WAITING_FOR_REMOTE_KVS - so = scheduler.schedule() - assert req.status == RequestStatus.WAITING_FOR_REMOTE_KVS - scheduler.update_from_output(so, EMPTY_MODEL_RUNNER_OUTPUT) - - # Recv completes - so = scheduler.schedule() - mro = copy.deepcopy(EMPTY_MODEL_RUNNER_OUTPUT) - mro.kv_connector_output = KVConnectorOutput(finished_recving={req_id}) - scheduler.update_from_output(so, mro) - - # Verify the flag is set - assert req.kv_transfer_params.get("_remote_blocks_processed") is True - - # Next schedule: update_state_after_alloc is called again. - # _reqs_need_recv must NOT contain this request. - so = scheduler.schedule() - assert req_id not in conn._reqs_need_recv - meta = so.kv_connector_metadata - assert isinstance(meta, NixlConnectorMetadata) - assert req_id not in meta.reqs_to_recv - - # Clean up - mro = create_model_runner_output(reqs=[req]) - scheduler.update_from_output(so, mro) - so = scheduler.schedule() - scheduler.update_from_output(so, EMPTY_MODEL_RUNNER_OUTPUT) - so = scheduler.schedule() - mro = copy.deepcopy(EMPTY_MODEL_RUNNER_OUTPUT) - mro.kv_connector_output = KVConnectorOutput(finished_sending={req_id}) - scheduler.update_from_output(so, mro) - assert_scheduler_empty(scheduler) - - -# 6. Turn-2 deadline expiry check (P declines reading expired D blocks) - -_REMOTE = FakeNixlConnectorWorker.REMOTE_ENGINE_ID - - -@pytest.mark.parametrize( - ("offset", "expiry_delta", "expect_declined"), - [ - # offset known + deadline expired/near-expiry -> decline & recompute. - pytest.param(0.0, -10.0, True, id="expired"), - # offset applied before comparison: +50s in D's clock is expired locally - # when D is 100s ahead. - pytest.param(100.0, 50.0, True, id="offset_makes_expired"), - # within the default 5s safety margin -> treated as expired. - pytest.param(0.0, 2.0, True, id="near_expiry_within_margin"), - # far-future deadline -> read proceeds. - pytest.param(0.0, 1000.0, False, id="valid_far_deadline"), - # no deadline field (older peer / router did not forward) -> skipped. - pytest.param(0.0, None, False, id="missing_expiry_field"), - ], -) -@patch( - "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper", - FakeNixlWrapper, -) -def test_turn2_deadline_gate(dist_init, offset, expiry_delta, expect_declined): - """P declines on the turn-2 readback if the deadline is known, - near-expiry and a handshake clock offset is known""" - connector, worker = _make_connector_with_fake_worker() - worker._engine_clock_offset[_REMOTE] = offset - expiry_time = None if expiry_delta is None else time.perf_counter() + expiry_delta - meta = NixlConnectorMetadata() - params = { - "do_remote_prefill": False, - "do_remote_decode": True, - "remote_block_ids": ([20, 21],), - "remote_engine_id": FakeNixlConnectorWorker.REMOTE_ENGINE_ID, - "remote_request_id": "decode-req", - "remote_host": "localhost", - "remote_port": 1234, - "remote_tp_size": 1, - } - if expiry_time is not None: - params["remote_blocks_expiry_time"] = expiry_time - meta.add_new_req_to_recv( - request_id="req", - local_block_ids=([10, 11],), - kv_transfer_params=params, - ) - _do_load_kv(connector, meta) - assert worker.xfer_stats.data["num_kv_expired_reqs"] == ( - [1] if expect_declined else [] - ) - # KV expiry must not pollute the transport failure counters. - assert worker.xfer_stats.data["num_failed_transfers"] == [] - assert worker.xfer_stats.data["num_failed_handshakes"] == [] - _, done_recving = connector.get_finished(finished_req_ids=set()) - assert "req" in done_recving - - -@patch( - "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper", - FakeNixlWrapper, -) -def test_turn2_full_prefix_hit_with_expired_deadline_skips_gate(dist_init): - """A full local prefix hit issues no READ. The expiry gate must be skipped - even when D's deadline is expired.""" - connector, worker = _make_connector_with_fake_worker() - worker._engine_clock_offset[_REMOTE] = 0.0 - - meta = NixlConnectorMetadata() - meta.add_new_req_to_recv( - request_id="req", - local_block_ids=(), # full prefix hit -> zero local block groups - kv_transfer_params={ - "do_remote_prefill": False, - "do_remote_decode": True, - "remote_block_ids": ([20, 21],), - "remote_engine_id": _REMOTE, - "remote_request_id": "decode-req", - "remote_host": "localhost", - "remote_port": 1234, - "remote_tp_size": 1, - # Expired on D's clock; with offset 0 it is expired locally too. - "remote_blocks_expiry_time": time.perf_counter() - 10.0, - }, - ) - # Must not raise, and the gate must not fire for a no-read prefix hit. - _do_load_kv(connector, meta) - assert worker.xfer_stats.data["num_kv_expired_reqs"] == [] - assert worker.xfer_stats.data["num_failed_transfers"] == [] - - -def test_d_node_request_finished_exports_blocks_expiry_time(): - """D-node (do_remote_prefill request) exports a future float expiry time.""" - vllm_config = create_vllm_config(kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG) - scheduler = create_scheduler(vllm_config) - BS = vllm_config.cache_config.block_size - req = create_request( - request_id=600, block_size=BS, num_tokens=int(BS * 2.5), do_remote_prefill=True - ) - scheduler.add_request(req) - req_id = req.request_id - so = scheduler.schedule() - scheduler.update_from_output( - so, create_model_runner_output(reqs=[], finished_recving={req_id}) - ) - so = scheduler.schedule() - eco = scheduler.update_from_output( - so, create_model_runner_output(reqs=[req], use_eos=True) - ) - kv = eco[0].outputs[0].kv_transfer_params - assert kv["do_remote_decode"] is True - assert isinstance(kv["remote_blocks_expiry_time"], float) - assert kv["remote_blocks_expiry_time"] > time.perf_counter() - - -def test_handshake_listener_appends_perf_counter_frame(): - """The handshake reply carries a 2nd frame with the listener's live - perf_counter.""" - import threading - - import msgspec - import zmq - - from vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_scheduler import ( - NixlBaseConnectorScheduler, - ) - from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import GET_META_MSG - from vllm.distributed.kv_transfer.kv_connector.v1.nixl.utils import zmq_ctx - from vllm.utils.network_utils import get_open_port, make_zmq_path - - encoded_data = {(0, 0): b"payload-rank-0"} - ready_event = threading.Event() - stop_event = threading.Event() - host = "127.0.0.1" - port = get_open_port() - listener = threading.Thread( - target=NixlBaseConnectorScheduler._nixl_handshake_listener, - args=(encoded_data, ready_event, stop_event, host, port), - daemon=True, - ) - listener.start() - try: - assert ready_event.wait(timeout=5) - path = make_zmq_path("tcp", host, port) - with zmq_ctx(zmq.REQ, path) as sock: # type: ignore[attr-defined] - sock.setsockopt(zmq.RCVTIMEO, 5000) # type: ignore[attr-defined] - t0 = time.perf_counter() - sock.send(msgspec.msgpack.encode((GET_META_MSG, 0, 0))) - parts = sock.recv_multipart() - t1 = time.perf_counter() - assert len(parts) == 2 - assert parts[0] == b"payload-rank-0" - remote_perf = msgspec.msgpack.decode(parts[1]) - assert isinstance(remote_perf, float) - # same-clock => offset is near zero. - offset = remote_perf - (t0 + t1) / 2 - assert abs(offset) < 1.0 - finally: - stop_event.set() - listener.join(timeout=5) diff --git a/tests/v1/kv_connector/unit/test_hf3fs_client.py b/tests/v1/kv_connector/unit/test_hf3fs_client.py deleted file mode 100644 index 26a5de11c5..0000000000 --- a/tests/v1/kv_connector/unit/test_hf3fs_client.py +++ /dev/null @@ -1,283 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Tests for resource management in hf3fs_client.py: constructor failure cleanup -and idempotent close(). Tests use mock to replace real I/O operations -(hf3fs_fuse.io, SharedMemory, os, CUDA). -Requires hf3fs_fuse.io to be installed; skipped otherwise. -""" - -from typing import Any -from unittest.mock import MagicMock, patch - -import pytest - -HF3FS_AVAILABLE = True -try: - from hf3fs_fuse.io import ( # noqa: F401 - deregister_fd, - extract_mount_point, - make_ioring, - make_iovec, - register_fd, - ) - - from vllm.distributed.kv_transfer.kv_connector.v1.hf3fs.hf3fs_client import ( - Hf3fsClient, - ) -except Exception: - HF3FS_AVAILABLE = False - -requires_hf3fs = pytest.mark.skipif( - not HF3FS_AVAILABLE, - reason="hf3fs_fuse.io is not available on this machine", -) - - -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- - - -class _FakeShm: - """Shared-memory stub matching the multiprocessing.shared_memory.SharedMemory - interface used by Hf3fsClient: - - Attributes accessed by the constructor: - .buf – memoryview / buffer-protocol object consumed by torch.frombuffer - Methods called during normal lifetime: - .unlink() – called right after the iovec is set up - .close() – called in _release_resources() - """ - - def __init__(self, size: int = 1024): - self._data = bytearray(size) - self.buf = memoryview(self._data) - self.closed = False - self.close_call_count = 0 - self.unlink_call_count = 0 - - def close(self): - self.closed = True - self.close_call_count += 1 - - def unlink(self): - self.unlink_call_count += 1 - - -# =========================================================================== -# TestHf3fsClientResourceManagement -# =========================================================================== - - -@requires_hf3fs -class TestHf3fsClientResourceManagement: - """Tests for constructor failure cleanup and idempotent close().""" - - _MOD = "vllm.distributed.kv_transfer.kv_connector.v1.hf3fs.hf3fs_client" - - # ------------------------------------------------------------------ - # Helper: build a minimal Hf3fsClient bypassing all real I/O so that - # we can fully control its internal state. - # ------------------------------------------------------------------ - - def _make_client(self, tmp_path): - """Return a fully-mocked Hf3fsClient with controllable internals.""" - fake_shm_r = _FakeShm() - fake_shm_w = _FakeShm() - - patcher_list: list[Any] = [ - patch(f"{self._MOD}.HF3FS_AVAILABLE", True), - patch(f"{self._MOD}.register_fd"), - patch(f"{self._MOD}.deregister_fd"), - patch(f"{self._MOD}.extract_mount_point", return_value="/mnt/hf3fs"), - patch(f"{self._MOD}.make_ioring", return_value=MagicMock()), - patch(f"{self._MOD}.make_iovec", return_value=MagicMock()), - patch( - "multiprocessing.shared_memory.SharedMemory", - side_effect=[fake_shm_r, fake_shm_w], - ), - patch("os.open", return_value=99), - patch("os.ftruncate"), - patch("os.close"), - patch("os.fsync"), - patch("torch.cuda.Stream", return_value=MagicMock()), - patch("torch.frombuffer", return_value=MagicMock()), - patch("torch.empty", return_value=MagicMock()), - ] - for p in patcher_list: - p.start() - - try: - client = Hf3fsClient( - path=str(tmp_path / "test.bin"), - size=1024, - bytes_per_page=256, - entries=4, - ) - finally: - for p in patcher_list: - p.stop() - - # Manually point internal handles to our controllable fakes so that - # assertions after close() can inspect them directly. - client.shm_r = fake_shm_r - client.shm_w = fake_shm_w - client.file = 99 - return client, fake_shm_r, fake_shm_w - - # ------------------------------------------------------------------ - # close() idempotency - # ------------------------------------------------------------------ - - def test_close_idempotent_and_handles_cleared(self, tmp_path): - """Multiple close() calls must not raise; deregister_fd called exactly - once, all handles set to None, shm.close() invoked.""" - client, shm_r, shm_w = self._make_client(tmp_path) - - with ( - patch(f"{self._MOD}.deregister_fd") as mock_dereg, - patch("os.close"), - ): - client.close() # first close - client.close() # second close — must be no-op - client.close() # third close — must be no-op - - assert client._closed is True - assert mock_dereg.call_count == 1, ( - f"deregister_fd called {mock_dereg.call_count} times; expected 1" - ) - for attr in ("iov_r", "iov_w", "ior_r", "ior_w", "shm_r", "shm_w", "file"): - assert getattr(client, attr) is None, f"{attr} should be None after close()" - assert shm_r.closed is True - assert shm_w.closed is True - - def test_flush_after_close_is_noop(self, tmp_path): - """flush() after close() must silently do nothing (no fsync call).""" - client, _, _ = self._make_client(tmp_path) - - with ( - patch(f"{self._MOD}.deregister_fd"), - patch("os.close"), - patch("os.fsync") as mock_fsync, - ): - client.close() - client.flush() - - mock_fsync.assert_not_called() - - # ------------------------------------------------------------------ - # Constructor failure leaves no leaked resources - # ------------------------------------------------------------------ - - def test_constructor_failure_after_file_open_cleans_file(self, tmp_path): - """If the constructor raises after os.open(), the fd must be closed.""" - with ( - patch(f"{self._MOD}.HF3FS_AVAILABLE", True), - patch(f"{self._MOD}.register_fd"), - patch(f"{self._MOD}.deregister_fd"), - patch( - f"{self._MOD}.extract_mount_point", - side_effect=RuntimeError("mount point not found"), - ), - patch("os.open", return_value=55), - patch("os.ftruncate"), - patch("os.close") as mock_os_close, - patch("torch.cuda.Stream", return_value=MagicMock()), - pytest.raises(RuntimeError, match="mount point not found"), - ): - Hf3fsClient( - path=str(tmp_path / "fail.bin"), - size=1024, - bytes_per_page=256, - entries=4, - ) - - mock_os_close.assert_called_once_with(55) - - def test_constructor_failure_after_shm_alloc_closes_shm(self, tmp_path): - """Constructor raises after SharedMemory creation → both shm objects closed.""" - fake_shm_r = _FakeShm() - fake_shm_w = _FakeShm() - - with ( - patch(f"{self._MOD}.HF3FS_AVAILABLE", True), - patch(f"{self._MOD}.register_fd"), - patch(f"{self._MOD}.deregister_fd"), - patch(f"{self._MOD}.extract_mount_point", return_value="/mnt/hf3fs"), - patch( - "multiprocessing.shared_memory.SharedMemory", - side_effect=[fake_shm_r, fake_shm_w], - ), - patch("os.open", return_value=66), - patch("os.ftruncate"), - patch("os.close"), - patch("torch.frombuffer", return_value=MagicMock()), - patch("torch.empty", return_value=MagicMock()), - patch( - f"{self._MOD}.make_ioring", - side_effect=RuntimeError("ioring init failed"), - ), - patch(f"{self._MOD}.make_iovec", return_value=MagicMock()), - patch("torch.cuda.Stream", return_value=MagicMock()), - pytest.raises(RuntimeError, match="ioring init failed"), - ): - Hf3fsClient( - path=str(tmp_path / "fail2.bin"), - size=1024, - bytes_per_page=256, - entries=4, - ) - - assert fake_shm_r.closed is True, ( - "shm_r was not closed after constructor failure" - ) - assert fake_shm_w.closed is True, ( - "shm_w was not closed after constructor failure" - ) - - def test_constructor_failure_does_not_close_unallocated_shm(self, tmp_path): - """Failure before SharedMemory is created must not raise AttributeError - or TypeError from cleanup.""" - with ( - patch(f"{self._MOD}.HF3FS_AVAILABLE", True), - patch(f"{self._MOD}.register_fd"), - patch(f"{self._MOD}.deregister_fd"), - patch( - f"{self._MOD}.extract_mount_point", - side_effect=RuntimeError("early failure"), - ), - patch("os.open", return_value=77), - patch("os.ftruncate"), - patch("os.close"), - patch("torch.cuda.Stream", return_value=MagicMock()), - pytest.raises(RuntimeError, match="early failure"), - ): - Hf3fsClient( - path=str(tmp_path / "early_fail.bin"), - size=1024, - bytes_per_page=256, - entries=4, - ) - - # ------------------------------------------------------------------ - # _release_resources on already-cleared state must be a no-op - # ------------------------------------------------------------------ - - def test_release_resources_on_empty_state_is_safe(self, tmp_path): - """_release_resources() on a fully-cleared client must not raise.""" - client, _, _ = self._make_client(tmp_path) - - with ( - patch(f"{self._MOD}.deregister_fd"), - patch("os.close"), - ): - client.close() # clears all handles - - with ( - patch(f"{self._MOD}.deregister_fd") as mock_dereg2, - patch("os.close") as mock_os_close2, - ): - client._release_resources() # must not raise - - mock_dereg2.assert_not_called() - mock_os_close2.assert_not_called() diff --git a/tests/v1/kv_connector/unit/test_hf3fs_connector.py b/tests/v1/kv_connector/unit/test_hf3fs_connector.py deleted file mode 100644 index de4d532798..0000000000 --- a/tests/v1/kv_connector/unit/test_hf3fs_connector.py +++ /dev/null @@ -1,233 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Tests for HF3FS KV Connector high-level components: -- TestHf3fsMockClient : file-backed mock client I/O correctness -- TestHF3FSKVConnectorStats: metric collection, aggregation, serialisation -""" - -import os -from unittest.mock import MagicMock - -import numpy as np -import pytest -import torch - -from vllm.distributed.kv_transfer.kv_connector.v1.hf3fs.hf3fs_connector import ( - HF3FSKVConnectorStats, -) -from vllm.distributed.kv_transfer.kv_connector.v1.hf3fs.utils.hf3fs_mock_client import ( - Hf3fsClient as MockHf3fsClient, -) - -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- - - -@pytest.fixture -def hf3fs_stats(): - """Fresh HF3FSKVConnectorStats instance.""" - return HF3FSKVConnectorStats() - - -def _make_cuda_event(): - """Return a real CUDA event when available, otherwise a MagicMock.""" - if torch.cuda.is_available(): - return torch.cuda.Event() - return MagicMock() - - -# =========================================================================== -# TestHf3fsMockClient -# =========================================================================== - - -class TestHf3fsMockClient: - """Tests for hf3fs_mock_client.Hf3fsClient (file-backend mock).""" - - def test_init_creates_file(self, tmp_path): - """Initializing the client should create the backing file.""" - path = str(tmp_path / "test_file") - client = MockHf3fsClient(path=path, size=4096, bytes_per_page=512, entries=4) - assert os.path.exists(path), "Backing file should be created on init" - assert os.path.getsize(path) == 4096 - client.close() - - @pytest.mark.parametrize( - "dtype, bytes_per_page", - [ - (torch.float32, 512), - (torch.float16, 256), - (torch.bfloat16, 256), - ], - ids=["float32", "float16", "bfloat16"], - ) - def test_batch_write_and_read_dtype(self, tmp_path, dtype, bytes_per_page): - """Write a tensor of the given dtype and verify round-trip correctness.""" - path = str(tmp_path / f"rw_{dtype}") - client = MockHf3fsClient( - path=path, size=bytes_per_page * 8, bytes_per_page=bytes_per_page, entries=4 - ) - elem_size = torch.tensor([], dtype=dtype).element_size() - numel = bytes_per_page // elem_size - tensor_write = torch.arange(numel, dtype=dtype) - event = _make_cuda_event() - - results = client.batch_write([0], [tensor_write], event) - assert results == [bytes_per_page], f"Write should succeed, got {results}" - - tensor_read = torch.zeros(numel, dtype=dtype) - results = client.batch_read([0], [tensor_read]) - assert results == [bytes_per_page], f"Read should succeed, got {results}" - assert torch.equal(tensor_write, tensor_read), ( - "Read tensor should match written tensor" - ) - client.close() - - def test_batch_read_empty_file_returns_error(self, tmp_path): - """Reading out-of-bounds offset should return -1.""" - bytes_per_page = 128 - size = bytes_per_page * 4 - path = str(tmp_path / "empty_read") - client = MockHf3fsClient( - path=path, size=size, bytes_per_page=bytes_per_page, entries=4 - ) - numel = bytes_per_page // 4 - tensor_read = torch.zeros(numel, dtype=torch.float32) - results = client.batch_read([size], [tensor_read]) # offset == size => OOB - assert results[0] == -1, "Out-of-bounds read should return -1" - client.close() - - def test_batch_write_out_of_bounds_returns_error(self, tmp_path): - """Writing at an offset beyond file size should return -1.""" - bytes_per_page = 128 - size = bytes_per_page * 4 - path = str(tmp_path / "oob_write") - client = MockHf3fsClient( - path=path, size=size, bytes_per_page=bytes_per_page, entries=4 - ) - numel = bytes_per_page // 4 - tensor = torch.ones(numel, dtype=torch.float32) - event = _make_cuda_event() - results = client.batch_write([size], [tensor], event) # OOB offset - assert results[0] == -1, "Out-of-bounds write should return -1" - client.close() - - def test_multiple_tensors_rw(self, tmp_path): - """Write multiple tensors at different offsets, then read all back.""" - bytes_per_page = 128 - n = 4 - path = str(tmp_path / "multi_rw") - client = MockHf3fsClient( - path=path, - size=bytes_per_page * n * 2, - bytes_per_page=bytes_per_page, - entries=8, - ) - tensors_write = [ - torch.full((bytes_per_page // 4,), float(i), dtype=torch.float32) - for i in range(n) - ] - offsets = [i * bytes_per_page for i in range(n)] - event = _make_cuda_event() - - results = client.batch_write(offsets, tensors_write, event) - assert all(r == bytes_per_page for r in results) - - tensors_read = [ - torch.zeros(bytes_per_page // 4, dtype=torch.float32) for _ in range(n) - ] - results = client.batch_read(offsets, tensors_read) - assert all(r == bytes_per_page for r in results) - - for i, (tw, tr) in enumerate(zip(tensors_write, tensors_read)): - assert torch.allclose(tw, tr), f"Tensor {i} mismatch after round-trip" - client.close() - - def test_flush_and_close_no_error(self, tmp_path): - """flush() and close() should not raise exceptions.""" - path = str(tmp_path / "flush_close") - client = MockHf3fsClient(path=path, size=1024, bytes_per_page=128, entries=4) - client.flush() - client.close() - - -# =========================================================================== -# TestHF3FSKVConnectorStats -# =========================================================================== - - -class TestHF3FSKVConnectorStats: - """Tests for HF3FSKVConnectorStats metric collection and aggregation.""" - - def test_initial_is_empty(self, hf3fs_stats): - """Fresh stats object should report is_empty() == True.""" - assert hf3fs_stats.is_empty() is True - - @pytest.mark.parametrize( - "task_type, duration_key", - [ - ("Saved", "save_duration"), - ("Loaded", "load_duration"), - ], - ids=["save", "load"], - ) - def test_record_success_duration(self, hf3fs_stats, task_type, duration_key): - """Recording a successful task should update duration list and total count.""" - hf3fs_stats.record_success_task_duration(task_type, 0.5) - assert not hf3fs_stats.is_empty() - assert len(hf3fs_stats.data[duration_key]) == 1 - assert hf3fs_stats.data[duration_key][0] == pytest.approx(0.5) - assert hf3fs_stats.data["num_transfer_task"] == 1 - - @pytest.mark.parametrize( - "task_type, failed_key", - [ - ("Saved", "num_failed_save"), - ("Loaded", "num_failed_load"), - ], - ids=["save", "load"], - ) - def test_record_failed_task(self, hf3fs_stats, task_type, failed_key): - """Recording a failed task should increment the corresponding counter.""" - hf3fs_stats.record_failed_task_count(task_type) - assert hf3fs_stats.data[failed_key] == 1 - assert hf3fs_stats.data["num_transfer_task"] == 1 - - def test_aggregate_two_stats(self): - """aggregate() should merge save/load duration lists and sum counters.""" - stats1 = HF3FSKVConnectorStats() - stats1.record_success_task_duration("Saved", 0.1) - stats1.record_success_task_duration("Loaded", 0.2) - - stats2 = HF3FSKVConnectorStats() - stats2.record_success_task_duration("Saved", 0.3) - stats2.record_failed_task_count("Loaded") - - stats1.aggregate(stats2) - assert stats1.data["save_duration"] == pytest.approx([0.1, 0.3]) - assert stats1.data["load_duration"] == pytest.approx([0.2]) - assert stats1.data["num_failed_load"] == 1 - assert stats1.data["num_transfer_task"] == 4 - - def test_reduce_with_data(self): - """reduce() computes correct averages when data is present.""" - stats = HF3FSKVConnectorStats() - stats.record_success_task_duration("Saved", 1.0) - stats.record_success_task_duration("Saved", 3.0) - result = stats.reduce() - assert result["Num save task success"] == pytest.approx(2.0, rel=0.01) - assert result["Num save task failed"] == pytest.approx(0.0, rel=0.01) - assert result["Avg save duration (ms)"] == pytest.approx(2000.0, rel=0.01) - # Reduced values must be plain Python scalars so CLI logging renders - # them without numpy reprs (eg np.float64(...)). - assert all(not isinstance(v, np.generic) for v in result.values()) - - def test_clone_and_reset(self, hf3fs_stats): - """clone_and_reset() returns a copy with data and resets the original.""" - hf3fs_stats.record_success_task_duration("Saved", 0.7) - hf3fs_stats.record_success_task_duration("Loaded", 0.4) - - clone = hf3fs_stats.clone_and_reset() - assert clone.data["num_transfer_task"] == 2 - assert hf3fs_stats.is_empty() diff --git a/tests/v1/kv_connector/unit/test_hf3fs_metadata_server.py b/tests/v1/kv_connector/unit/test_hf3fs_metadata_server.py deleted file mode 100644 index a366f95caa..0000000000 --- a/tests/v1/kv_connector/unit/test_hf3fs_metadata_server.py +++ /dev/null @@ -1,192 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Tests for HF3FS metadata server data structures and allocation logic: -- RankFileMetadata : page allocation / release primitives -- KeyMetadata : per-key rank-page tracking and completion detection -- GlobalMetadataState : coordinated allocation with cache-hit semantics -""" - -import pytest - -from vllm.distributed.kv_transfer.kv_connector.v1.hf3fs.hf3fs_metadata_server import ( - GlobalMetadataState, - KeyMetadata, - RankFileMetadata, -) - -# =========================================================================== -# TestRankFileMetadata -# =========================================================================== - - -class TestRankFileMetadata: - """Unit tests for RankFileMetadata page allocation primitives.""" - - @pytest.mark.parametrize( - "alloc_count, expected_pages", - [(3, 3), (5, 0)], - ids=["alloc_partial", "alloc_exceeds"], - ) - def test_allocate_pages(self, alloc_count, expected_pages): - """allocate_pages returns correct pages or empty list when insufficient.""" - rank_meta = RankFileMetadata(rank_id=0, num_pages=3, free_pages=list(range(3))) - pages = rank_meta.allocate_pages(alloc_count) - assert len(pages) == expected_pages - if expected_pages > 0: - rank_meta.release_pages(pages) - assert rank_meta.get_free_page_count() == 3 - - def test_release_pages_restores_count(self): - """Releasing allocated pages returns them to the free pool.""" - rank_meta = RankFileMetadata(rank_id=0, num_pages=4, free_pages=list(range(4))) - pages = rank_meta.allocate_pages(2) - assert rank_meta.get_free_page_count() == 2 - rank_meta.release_pages(pages) - assert rank_meta.get_free_page_count() == 4 - - def test_release_pages_no_duplicates(self): - """Releasing the same page twice must not create duplicates.""" - rank_meta = RankFileMetadata(rank_id=0, num_pages=3, free_pages=list(range(3))) - rank_meta.allocate_pages(1) # takes page 0 - rank_meta.release_pages([0]) - rank_meta.release_pages([0]) # second release of the same page - assert rank_meta.get_free_page_count() == 3 - - -# =========================================================================== -# TestKeyMetadata -# =========================================================================== - - -class TestKeyMetadata: - """Unit tests for KeyMetadata completion tracking.""" - - def test_is_complete_false_until_all_ranks(self): - """is_complete() returns True only when all ranks confirmed.""" - key_meta = KeyMetadata(key="k", rank_to_page={}, tp_world_size=2) - assert key_meta.is_complete() is False - key_meta.add_rank_page(0, 5) - assert key_meta.is_complete() is False - key_meta.add_rank_page(1, 10) - assert key_meta.is_complete() is True - - def test_get_rank_page_returns_none_for_missing_rank(self): - """get_rank_page() returns None when the rank has no entry.""" - key_meta = KeyMetadata(key="k", rank_to_page={0: 3}, tp_world_size=2) - assert key_meta.get_rank_page(0) == 3 - assert key_meta.get_rank_page(1) is None - - def test_get_all_pages(self): - """get_all_pages() returns all (rank, page) pairs.""" - key_meta = KeyMetadata(key="k", rank_to_page={0: 1, 1: 2}, tp_world_size=2) - pairs = key_meta.get_all_pages() - assert set(pairs) == {(0, 1), (1, 2)} - - -# =========================================================================== -# TestGlobalMetadataStateAllocation -# =========================================================================== - - -class TestGlobalMetadataStateAllocation: - """Tests for GlobalMetadataState allocation and cache-hit semantics.""" - - def test_uninitialized_rank_raises_on_allocate(self): - """allocate_pages_for_keys raises ValueError for unknown rank.""" - state = GlobalMetadataState() - with pytest.raises((ValueError, Exception)): - state.allocate_pages_for_keys(99, [("key", "")]) - - def test_uninitialized_rank_raises_on_get_locations(self): - """get_key_locations raises ValueError for unknown rank.""" - state = GlobalMetadataState() - with pytest.raises((ValueError, Exception)): - state.get_key_locations(99, ["any_key"]) - - def test_basic_allocation_and_confirm(self): - """Allocating a page and confirming it marks the key as complete.""" - state = GlobalMetadataState() - state.initialize_rank(0, 4) - - results = state.allocate_pages_for_keys(0, [("K", "")]) - assert results["K"] >= 0 - - state.confirm_write_for_keys(0, [("K", results["K"])]) - assert state.batch_key_exists(["K"]) == [True] - locations = state.get_key_locations(0, ["K"]) - assert locations == [results["K"]] - - def test_allocate_pages_cache_hit_does_not_leak_pages(self): - """Cache-hit key must not consume a page from the free pool; - the pre-allocated slot must be returned before reusing the existing page. - """ - state = GlobalMetadataState() - state.initialize_rank(0, 5) # 5 free pages: [0,1,2,3,4] - - # Simulate a key that has already been fully written and confirmed. - state.key_metadata["K_cached"] = KeyMetadata( - key="K_cached", rank_to_page={0: 2}, tp_world_size=1 - ) - - free_before = state.rank_metadata[0].get_free_page_count() # 5 - - results = state.allocate_pages_for_keys(0, [("K_cached", ""), ("K_new", "")]) - - free_after = state.rank_metadata[0].get_free_page_count() - - # Cache-hit key must reuse its existing page. - assert results["K_cached"] == 2, ( - f"Cache-hit key should reuse page 2, got {results['K_cached']}" - ) - # New key must receive a valid page. - assert results["K_new"] >= 0, ( - f"New key should get a valid page, got {results['K_new']}" - ) - # Exactly one page consumed from the free pool. - assert free_before - free_after == 1, ( - f"Expected 1 page consumed, got delta={free_before - free_after}" - ) - - def test_allocate_pages_all_cache_hits_frees_all_slots(self): - """When every key in the batch is a cache hit, no pages are consumed.""" - state = GlobalMetadataState() - state.initialize_rank(0, 5) - - for key, page in (("K1", 0), ("K2", 1)): - state.key_metadata[key] = KeyMetadata( - key=key, rank_to_page={0: page}, tp_world_size=1 - ) - - free_before = state.rank_metadata[0].get_free_page_count() - results = state.allocate_pages_for_keys(0, [("K1", ""), ("K2", "")]) - free_after = state.rank_metadata[0].get_free_page_count() - - assert results["K1"] == 0 - assert results["K2"] == 1 - assert free_after == free_before, ( - f"All-cache-hit batch must not consume free pages; " - f"before={free_before}, after={free_after}" - ) - - def test_allocate_returns_minus_one_when_pool_exhausted(self): - """If the free pool is exhausted, all new keys receive -1.""" - state = GlobalMetadataState() - state.initialize_rank(0, 1) # only 1 free page - - results = state.allocate_pages_for_keys(0, [("K1", ""), ("K2", "")]) - # allocate_pages uses all-or-nothing: 2 needed but only 1 available → [] - assert all(v == -1 for v in results.values()), f"Expected all -1, got {results}" - - def test_confirm_write_releases_pages(self): - """confirm_write_for_keys with pages_to_release returns them to pool.""" - state = GlobalMetadataState() - state.initialize_rank(0, 3) - - results = state.allocate_pages_for_keys(0, [("K", "")]) - page = results["K"] - free_after_alloc = state.rank_metadata[0].get_free_page_count() - - state.confirm_write_for_keys(0, [("K", page)], pages_to_release=[page]) - free_after_release = state.rank_metadata[0].get_free_page_count() - - assert free_after_release == free_after_alloc + 1 diff --git a/tests/v1/kv_connector/unit/test_hisparse_stats.py b/tests/v1/kv_connector/unit/test_hisparse_stats.py deleted file mode 100644 index f0b09ecf36..0000000000 --- a/tests/v1/kv_connector/unit/test_hisparse_stats.py +++ /dev/null @@ -1,158 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -import pytest - -from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.connector import ( - HiSparseConnector, -) -from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.stats import ( - HiSparseKVConnectorStats, -) - - -def test_is_empty_on_fresh_stats(): - stats = HiSparseKVConnectorStats() - assert stats.is_empty() - - -def test_record_snapshot_and_reduce(): - stats = HiSparseKVConnectorStats() - stats.record_snapshot(hits=7, misses=3, host_to_device_bytes=48) - stats.record_snapshot(hits=5, misses=1, host_to_device_bytes=16) - assert not stats.is_empty() - - reduced = stats.reduce() - assert reduced["HiSparse hot-buffer hits"] == 12 - assert reduced["HiSparse hot-buffer misses"] == 4 - assert reduced["HiSparse host-to-device bytes"] == 64 - - -def test_record_host_usage_reports_latest_level(): - stats = HiSparseKVConnectorStats() - stats.record_host_usage(usage=0.4, pending_page_transfers=2) - stats.record_host_usage(usage=0.8, pending_page_transfers=0) - - reduced = stats.reduce() - assert reduced["HiSparse host KV cache usage %"] == 80.0 - - -def test_aggregate_keeps_latest_host_usage_level(): - first = HiSparseKVConnectorStats() - first.record_host_usage(usage=0.4, pending_page_transfers=2) - second = HiSparseKVConnectorStats() - second.record_host_usage(usage=0.8, pending_page_transfers=0) - second.record_host_usage(usage=0.6, pending_page_transfers=1) - - first.aggregate(second) - - assert first.data["host_cache_usage_perc"] == [0.4, 0.6] - assert first.data["pending_page_transfers"] == [2, 1] - - -def test_aggregate_extends_snapshot_deltas(): - first = HiSparseKVConnectorStats() - first.record_snapshot(hits=7, misses=3, host_to_device_bytes=48) - second = HiSparseKVConnectorStats() - second.record_snapshot(hits=5, misses=1, host_to_device_bytes=16) - - first.aggregate(second) - - assert first.data["cache_hits"] == [7, 5] - assert first.data["cache_misses"] == [3, 1] - assert first.data["host_to_device_bytes"] == [48, 16] - - -def test_aggregate_skips_empty_stats(): - stats = HiSparseKVConnectorStats() - stats.record_snapshot(hits=2, misses=1, host_to_device_bytes=16) - - stats.aggregate(HiSparseKVConnectorStats()) - - assert stats.data["cache_hits"] == [2] - - -def test_build_kv_connector_stats_round_trip(): - stats = HiSparseKVConnectorStats() - stats.record_snapshot(hits=12, misses=4, host_to_device_bytes=64) - payload = stats.to_dict() - - rebuilt = HiSparseConnector.build_kv_connector_stats(data=payload) - - assert rebuilt is not None - assert isinstance(rebuilt, HiSparseKVConnectorStats) - assert rebuilt.reduce() == stats.reduce() - - -@pytest.mark.parametrize("kv_cache_metrics", [True, False]) -def test_prom_metrics_observe_host_metrics(kv_cache_metrics: bool): - from types import SimpleNamespace - from typing import Any - - from prometheus_client import Counter, Gauge, Histogram - - from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.stats import ( - HiSparsePromMetrics, - ) - from vllm.v1.metrics.stats import KVCacheEvictionEvent - - class _FakeMetric: - def __init__(self, **kwargs: Any): - self.kwargs = kwargs - self.increments: list[int | float] = [] - self.set_values: list[int | float] = [] - self.observed: list[float] = [] - - def labels(self, *labelvalues: object) -> "_FakeMetric": - return self - - def inc(self, value: int | float) -> None: - self.increments.append(value) - - def set(self, value: int | float) -> None: - self.set_values.append(value) - - def observe(self, value: float) -> None: - self.observed.append(value) - - created: dict[str, _FakeMetric] = {} - - class _NamedFake(_FakeMetric): - def __init__(self, **kwargs: Any): - super().__init__(**kwargs) - created[kwargs["name"]] = self - - prom = HiSparsePromMetrics( - vllm_config=SimpleNamespace( - kv_transfer_config=None, - observability_config=SimpleNamespace( - kv_cache_metrics=kv_cache_metrics, custom_histogram_buckets=None - ), - ), - metric_types={Gauge: _NamedFake, Counter: _NamedFake, Histogram: _NamedFake}, - labelnames=["model_name"], - per_engine_labelvalues={0: ["model"]}, - ) - - stats = HiSparseKVConnectorStats() - stats.record_snapshot(hits=3, misses=2, host_to_device_bytes=32) - stats.record_host_usage(usage=0.5, pending_page_transfers=1) - stats.record_host_usage(usage=0.75, pending_page_transfers=3) - stats.record_host_evictions([KVCacheEvictionEvent(5.0, 2.0, (1.0, 2.0))]) - later = HiSparseKVConnectorStats() - later.record_host_evictions([KVCacheEvictionEvent(3.0, 3.0, ())]) - prom.observe(stats.aggregate(later).to_dict()) - - assert created["vllm:hisparse_cache_hits"].increments == [3] - assert created["vllm:hisparse_host_to_device_bytes"].increments == [32] - assert created["vllm:hisparse_host_cache_usage_perc"].set_values == [0.75] - assert created["vllm:hisparse_pending_page_transfers"].set_values == [3] - if not kv_cache_metrics: - assert not any("host_block" in name for name in created) - return - assert created["vllm:hisparse_host_block_lifetime_seconds"].observed == [5.0, 3.0] - assert created["vllm:hisparse_host_block_idle_before_evict_seconds"].observed == [ - 2.0, - 3.0, - ] - assert created["vllm:hisparse_host_block_reuse_gap_seconds"].observed == [1.0, 2.0] diff --git a/tests/v1/kv_connector/unit/test_moriio_hma_scheduler.py b/tests/v1/kv_connector/unit/test_moriio_hma_scheduler.py deleted file mode 100644 index 1120062e54..0000000000 --- a/tests/v1/kv_connector/unit/test_moriio_hma_scheduler.py +++ /dev/null @@ -1,1414 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Pure-CPU unit tests for the MoRIIO hybrid (attention + mamba/KDA) scheduler -and the KDA offset-template cache. - -These exercise the highest-risk, otherwise-untested scheduler logic without a -GPU or the ``mori`` runtime: per-group block-id splitting, the READ/WRITE -``N-1`` token accounting, P-side prompt truncation, and offset-template cache -wiring. -""" - -import importlib -from dataclasses import asdict -from types import SimpleNamespace -from unittest.mock import Mock - -import pytest -import torch - -from vllm.v1.core.block_pool import BlockPool -from vllm.v1.core.sched.output import KVConnectorBlockState, SchedulerOutput -from vllm.v1.core.single_type_kv_cache_manager import FullAttentionManager -from vllm.v1.kv_cache_interface import FullAttentionSpec, UniformTypeKVCacheSpecs - -moriio_connector = importlib.import_module( - "vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_connector" -) -moriio_layout = importlib.import_module( - "vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_layout" -) -ssm_conv_transfer_utils = importlib.import_module( - "vllm.distributed.kv_transfer.kv_connector.v1.ssm_conv_transfer_utils" -) -MambaConvSplitInfo = ssm_conv_transfer_utils.MambaConvSplitInfo -MoRIIOMode = moriio_connector.MoRIIOMode -moriio_common = importlib.import_module( - "vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_common" -) -get_role = moriio_common.get_role -set_role = moriio_common.set_role - - -@pytest.fixture -def restore_moriio_role(): - """Undo the process-global role a full scheduler construction leaves behind.""" - role = get_role() - yield - set_role(role) - - -class _FakeScheduler(moriio_connector.MoRIIOConnectorScheduler): # type: ignore[name-defined] - """Constructs a scheduler without the heavy real ``__init__``; only the - attributes read by the method under test are set.""" - - def __init__(self, **attrs): - self.mode = MoRIIOMode.READ - self._mamba_group_ids: list[int] = [] - self._attn_group_ids: list[int] = [0] - self._num_ssm_scratch_blocks = 0 - self._is_hma_required = False - self.kv_cache_config = SimpleNamespace( - kv_cache_groups=[None, None], - select_transfer_block_ids=lambda block_ids: tuple(block_ids), - ) - for k, v in attrs.items(): - setattr(self, k, v) - - -class _FakeConnector(moriio_connector.MoRIIOConnector): # type: ignore[name-defined] - def __init__(self, connector_scheduler): - self.connector_scheduler = connector_scheduler - - -class _FakeWorker(moriio_connector.MoRIIOConnectorWorker): # type: ignore[name-defined] - def __init__(self, **attrs): - for k, v in attrs.items(): - setattr(self, k, v) - - -class _FakeBlocks: - def __init__(self, all_groups): - self._all_groups = all_groups - - def get_block_ids(self): - return self._all_groups - - -def _slot_strided(num_slots, per_slot_shape, slot_stride, dtype=torch.bfloat16): - inner = 1 - for d in per_slot_shape: - inner *= d - assert slot_stride >= inner - backing = torch.zeros(num_slots * slot_stride, dtype=dtype) - inner_strides: list[int] = [] - acc = 1 - for d in reversed(per_slot_shape): - inner_strides.insert(0, acc) - acc *= d - return backing.as_strided( - (num_slots, *per_slot_shape), (slot_stride, *inner_strides) - ) - - -def _gdn_split_info(conv_rows=3, key_dim=4, value_dim=8, dtype_size=2): - conv_dim = 2 * key_dim + value_dim - conv_state_bytes = conv_dim * conv_rows * dtype_size - ssm_state_bytes = 64 - return MambaConvSplitInfo( - conv_rows=conv_rows, - local_proj_dims=(key_dim, key_dim, value_dim), - conv_dtype_size=dtype_size, - ssm_sizes=(conv_state_bytes, ssm_state_bytes), - ) - - -def _mamba_spec(num_states: int = 2, *, num_speculative_blocks: int = 0): - shapes = tuple((1, 1) for _ in range(num_states)) - return moriio_connector.MambaSpec( - block_size=16, - shapes=shapes, - dtypes=(torch.float32,) * num_states, - mamba_cache_mode="align", - num_speculative_blocks=num_speculative_blocks, - ) - - -def _spec_config(method: str): - return SimpleNamespace(method=method, use_dspark=lambda: method == "dspark") - - -def _gate_vllm_config( - *, - read_mode: bool = True, - speculative_config=None, - block_size: int = 16, - num_lookahead_tokens: int = 0, - num_speculative_tokens: int = 0, - mamba_cache_mode: str = "align", - dcp_size: int = 1, -): - return SimpleNamespace( - kv_transfer_config=SimpleNamespace( - kv_connector_extra_config={ - "read_mode": read_mode, - "host_ip": "127.0.0.1", - "handshake_port": 6001, - "notify_port": 6002, - }, - kv_role="kv_consumer", - ), - speculative_config=speculative_config, - cache_config=SimpleNamespace( - block_size=block_size, - mamba_cache_mode=mamba_cache_mode, - ), - scheduler_config=SimpleNamespace(disable_hybrid_kv_cache_manager=False), - num_lookahead_tokens=num_lookahead_tokens, - num_speculative_tokens=num_speculative_tokens, - parallel_config=SimpleNamespace( - tensor_parallel_size=1, - decode_context_parallel_size=dcp_size, - data_parallel_rank=0, - data_parallel_size_local=1, - data_parallel_size=1, - ), - ) - - -# -------------------------------------------------------------------------- -# split_block_groups -# -------------------------------------------------------------------------- -def test_split_block_groups_separates_attention_and_mamba(): - sched = _FakeScheduler(_has_mamba=True, _attn_group_ids=[0], _mamba_group_ids=[1]) - block_ids = ([1, 2], [99]) - attn, mamba = sched.split_block_groups(block_ids) - assert attn == [1, 2] - assert mamba == [[99]] - - -def test_split_block_groups_ignores_transfer_disabled_group(): - mamba_spec = moriio_connector.MambaSpec( - block_size=16, - shapes=((1, 1),), - dtypes=(torch.float32,), - mamba_cache_mode="align", - ) - config = SimpleNamespace( - kv_cache_groups=[ - SimpleNamespace(enable_kv_transfer=True, kv_cache_spec=object()), - SimpleNamespace(enable_kv_transfer=False, kv_cache_spec=object()), - SimpleNamespace(enable_kv_transfer=True, kv_cache_spec=mamba_spec), - ], - ) - config.transfer_groups = tuple( - group for group in config.kv_cache_groups if group.enable_kv_transfer - ) - config.select_transfer_block_ids = lambda block_ids: tuple( - block_ids[group_id] - for group_id, group in enumerate(config.kv_cache_groups) - if group.enable_kv_transfer - ) - attn_groups, mamba_groups = moriio_connector._split_kv_cache_group_kinds(config) - assert attn_groups == [0] - assert mamba_groups == [1] - - sched = _FakeScheduler( - _has_mamba=True, - _attn_group_ids=attn_groups, - _mamba_group_ids=mamba_groups, - kv_cache_config=config, - ) - assert sched.split_block_groups(([1, 2], [70, 71], [99])) == ( - [1, 2], - [[99]], - ) - - -def test_exchange_blocks_ignore_transfer_disabled_group(): - config = SimpleNamespace( - kv_cache_groups=[None, None, None], - select_transfer_block_ids=lambda block_ids: (block_ids[0], block_ids[2]), - ) - sched = _FakeScheduler( - kv_cache_config=config, - _is_hma_required=True, - blocks_per_sw=[0, 2], - ) - - assert sched.get_exchange_clipped_blocks(([1, 2], [70, 71], [90, 91, 92])) == [ - [1, 2], - [91, 92], - ] - - -def test_scheduler_rejects_multiple_attention_groups_with_mamba(): - """A drafter owning its own transfer group is refused, with a hint.""" - config = SimpleNamespace( - kv_cache_groups=[ - SimpleNamespace(enable_kv_transfer=True, kv_cache_spec=object()), - SimpleNamespace(enable_kv_transfer=True, kv_cache_spec=object()), - SimpleNamespace(enable_kv_transfer=True, kv_cache_spec=_mamba_spec()), - ], - ) - config.transfer_groups = tuple(config.kv_cache_groups) - - with pytest.raises(moriio_common.MoRIIOError, match="separate attention group"): - moriio_connector.MoRIIOConnectorScheduler(_gate_vllm_config(), "engine", config) - - -def test_split_block_groups_preserves_multiple_mamba_groups(): - sched = _FakeScheduler( - _has_mamba=True, - _attn_group_ids=[0], - _mamba_group_ids=[1, 2], - ) - - assert sched.split_block_groups(([1, 2], [40], [70])) == ( - [1, 2], - [[40], [70]], - ) - - -def test_scheduler_rejects_incompatible_mamba_groups(): - incompatible = _mamba_spec() - object.__setattr__(incompatible, "page_size_padded", 4096) - config = SimpleNamespace( - kv_cache_groups=[ - SimpleNamespace(enable_kv_transfer=True, kv_cache_spec=object()), - SimpleNamespace(enable_kv_transfer=True, kv_cache_spec=_mamba_spec()), - SimpleNamespace(enable_kv_transfer=True, kv_cache_spec=incompatible), - ] - ) - config.transfer_groups = tuple(config.kv_cache_groups) - - with pytest.raises(moriio_common.MoRIIOError, match="same cache spec"): - moriio_connector.MoRIIOConnectorScheduler(_gate_vllm_config(), "engine", config) - - -def test_scheduler_rejects_mamba_group_without_two_states(): - config = SimpleNamespace( - kv_cache_groups=[ - SimpleNamespace(enable_kv_transfer=True, kv_cache_spec=object()), - SimpleNamespace(enable_kv_transfer=True, kv_cache_spec=_mamba_spec(1)), - ], - ) - config.transfer_groups = tuple(config.kv_cache_groups) - - with pytest.raises(moriio_common.MoRIIOError, match="exactly two Mamba states"): - moriio_connector.MoRIIOConnectorScheduler(_gate_vllm_config(), "engine", config) - - -@pytest.mark.parametrize("read_mode", [True, False]) -def test_scheduler_rejects_non_dspark_speculative_hybrid(read_mode): - config = SimpleNamespace( - kv_cache_groups=[ - SimpleNamespace(enable_kv_transfer=True, kv_cache_spec=object()), - SimpleNamespace(enable_kv_transfer=True, kv_cache_spec=_mamba_spec()), - ], - ) - config.transfer_groups = tuple(config.kv_cache_groups) - - with pytest.raises(moriio_common.MoRIIOError, match="speculative decoding"): - moriio_connector.MoRIIOConnectorScheduler( - _gate_vllm_config( - read_mode=read_mode, speculative_config=_spec_config("ngram") - ), - "engine", - config, - ) - - -def _dspark_wrapped_attention_config(*, dcp_sharded: bool = True): - """Mamba first, so a wrapped full-attention group at index 1 has to be found. - - A bare ``isinstance(FullAttentionSpec)`` misses the wrapper and leaves - ``_full_attn_group_idx`` at its initial 0, which is the Mamba group. - """ - full = FullAttentionSpec( - block_size=4, - num_kv_heads=1, - head_size=1, - dtype=torch.float32, - dcp_sharded=dcp_sharded, - ) - config = SimpleNamespace( - kv_cache_groups=[ - SimpleNamespace( - enable_kv_transfer=True, - kv_cache_spec=_mamba_spec(num_speculative_blocks=2), - ), - SimpleNamespace( - enable_kv_transfer=True, - kv_cache_spec=UniformTypeKVCacheSpecs( - block_size=4, kv_cache_specs={"target": full, "draft": full} - ), - ), - ], - ) - config.transfer_groups = tuple(config.kv_cache_groups) - return config - - -@pytest.mark.usefixtures("restore_moriio_role") -@pytest.mark.parametrize("read_mode", [True, False]) -@pytest.mark.parametrize( - "dcp_size,dcp_sharded,expected_tail", [(1, True, 4), (4, True, 1), (4, False, 4)] -) -def test_scheduler_accepts_dspark_with_wrapped_full_attention( - dcp_size, dcp_sharded, expected_tail, read_mode -): - config = _dspark_wrapped_attention_config(dcp_sharded=dcp_sharded) - vllm_config = _gate_vllm_config( - read_mode=read_mode, - speculative_config=_spec_config("dspark"), - block_size=4, - num_lookahead_tokens=7, - num_speculative_tokens=7, - dcp_size=dcp_size, - ) - scheduler = moriio_connector.MoRIIOConnectorScheduler(vllm_config, "engine", config) - - assert scheduler._full_attn_group_idx == 1 - assert scheduler._num_ssm_scratch_blocks == 2 - assert scheduler._max_decode_tail_blocks == expected_tail - - # A first decode step reserves padded target rows AND drafter lookahead. - spec = config.transfer_groups[1].kv_cache_spec - manager = FullAttentionManager( - spec, - block_pool=BlockPool(32, enable_caching=False, hash_block_size=4), - enable_caching=False, - kv_cache_group_id=1, - scheduler_block_size=16, - dcp_world_size=dcp_size, - ) - imported_tokens = manager.block_size - target_tokens = 1 + vllm_config.num_speculative_tokens - manager.allocate_new_blocks( - "decode", - num_tokens=imported_tokens + target_tokens + vllm_config.num_lookahead_tokens, - num_tokens_main_model=imported_tokens + target_tokens, - ) - local = [b.block_id for b in manager.req_to_blocks["decode"]] - remote = [100] - assert scheduler._align_read_blocks( - local, remote, scheduler._max_decode_tail_blocks - ) == (local[:1], remote) - with pytest.raises(ValueError, match="tail"): - scheduler._align_read_blocks( - local + [99], remote, scheduler._max_decode_tail_blocks - ) - - -@pytest.mark.usefixtures("restore_moriio_role") -def test_hybrid_write_clamps_attention_and_excludes_scratch_slots(): - config = _dspark_wrapped_attention_config() - config.select_transfer_block_ids = lambda blocks: tuple(blocks) - sched = moriio_connector.MoRIIOConnectorScheduler( - _gate_vllm_config(read_mode=False, dcp_size=2), "engine", config - ) - assert sched._clamp_to_prompt_blocks( - SimpleNamespace(num_prompt_tokens=9), [[10, 11, 12, 13], [1, 2, 3, 4]] - ) == [[11], [1, 2]] - - -@pytest.mark.usefixtures("restore_moriio_role") -def test_hybrid_write_resolves_current_state_instead_of_cached_block_ids(): - config = _dspark_wrapped_attention_config() - config.select_transfer_block_ids = lambda blocks: tuple(blocks) - sched = moriio_connector.MoRIIOConnectorScheduler( - _gate_vllm_config(read_mode=False, dcp_size=2), "engine", config - ) - state = KVConnectorBlockState( - req_ids={"req"}, - resolve_block_ids=lambda _: ([0, 51, 52, 53], [31, 32, 33]), - boundary_state_offloads={}, - ) - assert sched._get_write_blocks( - SimpleNamespace(request_id="req", num_prompt_tokens=9), - [[10, 11, 12], [1]], - SimpleNamespace(kv_connector_block_state=state), - ) == [[51], [31, 32]] - - -@pytest.mark.usefixtures("restore_moriio_role") -@pytest.mark.parametrize("external_tokens", [0, 9]) -def test_hybrid_write_notifies_grouped_slots_or_releases_a_complete_local_hit( - external_tokens, -): - config = _dspark_wrapped_attention_config() - config.select_transfer_block_ids = lambda blocks: tuple(blocks) - sched = moriio_connector.MoRIIOConnectorScheduler( - _gate_vllm_config(read_mode=False), "engine", config - ) - sched.send_notify_block = Mock() - sched._send_transfer_release = Mock() - req = _mk_request( - list(range(10)), - params={ - "do_remote_prefill": True, - "transfer_id": "tx", - "remote_host": "127.0.0.2", - "remote_notify_port": 7000, - }, - ) - req.request_id = "req" - sched.update_state_after_alloc( - req, _FakeBlocks(([10, 11, 12], [1, 2, 3])), external_tokens - ) - if external_tokens: - sched.send_notify_block.assert_called_once_with( - req_id="req", - transfer_id="tx", - block_notify_list=[[10], [1, 2, 3]], - host="127.0.0.2", - port=7000, - ) - sched._send_transfer_release.assert_not_called() - else: - sched.send_notify_block.assert_not_called() - sched._send_transfer_release.assert_called_once_with("tx", "127.0.0.2", 7000) - assert not req.kv_transfer_params["do_remote_prefill"] - - -@pytest.mark.usefixtures("restore_moriio_role") -def test_hybrid_write_keeps_separate_draft_groups_and_their_block_spans(): - config = _dspark_wrapped_attention_config() - # A disabled cache is absent from the wire. Target and draft may have - # different token spans, and neither can borrow the other's block IDs. - draft = SimpleNamespace( - enable_kv_transfer=True, - kv_cache_spec=FullAttentionSpec( - block_size=4, - num_kv_heads=1, - head_size=2, - dtype=torch.float32, - dcp_sharded=False, - ), - ) - disabled = SimpleNamespace(enable_kv_transfer=False, kv_cache_spec=object()) - config.kv_cache_groups.insert(1, disabled) - config.kv_cache_groups.append(draft) - config.transfer_groups = tuple( - g for g in config.kv_cache_groups if g.enable_kv_transfer - ) - config.select_transfer_block_ids = lambda blocks: (blocks[0], blocks[2], blocks[3]) - sched = moriio_connector.MoRIIOConnectorScheduler( - _gate_vllm_config( - read_mode=False, dcp_size=2, speculative_config=_spec_config("dspark") - ), - "engine", - config, - ) - req = _mk_request( - list(range(9)), - params={ - "do_remote_decode": True, - "transfer_id": "tx", - }, - ) - req.request_id = "req" - req.status = moriio_connector.RequestStatus.FINISHED_LENGTH_CAPPED - blocks = ([10, 11, 12, 13], [99], [20, 21, 22], [30, 31, 32, 33]) - expected = [[11], [20, 21], [30, 31, 32]] - assert sched._clamp_to_prompt_blocks(req, blocks) == expected - # Exercise the connector entry point too: completion must not flatten - # a multi-group WRITE payload through READ's single-attention helper. - delay_free, params = _FakeConnector(sched).request_finished_all_groups(req, blocks) - assert delay_free - assert params["remote_block_ids"] == expected - - -@pytest.mark.usefixtures("restore_moriio_role") -@pytest.mark.parametrize( - "blocks,delay_free", - [(([0, 51, 52, 53], [31, 32, 33]), True), (([], []), False)], -) -def test_hybrid_write_request_finished_defers_free_only_with_blocks_to_send( - blocks, delay_free -): - config = _dspark_wrapped_attention_config() - config.select_transfer_block_ids = lambda blocks: tuple(blocks) - sched = moriio_connector.MoRIIOConnectorScheduler( - _gate_vllm_config(read_mode=False, dcp_size=2), "engine", config - ) - sched.is_producer = True - req = _mk_request( - list(range(9)), params={"do_remote_decode": True, "transfer_id": "tx"} - ) - req.request_id = "req" - req.status = moriio_connector.RequestStatus.FINISHED_LENGTH_CAPPED - - assert sched.request_finished(req, blocks)[0] is delay_free - # The WRITE ACK, not the READ watchdog, releases the deferred blocks. - if delay_free: - assert sched._deferred_send_deadlines["req"][1] == "tx" - else: - assert "req" not in sched._deferred_send_deadlines - - -@pytest.mark.usefixtures("restore_moriio_role") -def test_hybrid_write_build_meta_saves_current_state_on_the_final_chunk(): - """Chunked prefill must not ship block ids cached at admission: align-mode - state slots move between chunks without appending a block.""" - config = _dspark_wrapped_attention_config() - config.select_transfer_block_ids = lambda blocks: tuple(blocks) - sched = moriio_connector.MoRIIOConnectorScheduler( - _gate_vllm_config(read_mode=False, dcp_size=2), "engine", config - ) - sched.is_producer = True - set_role(moriio_common.ROLE.PRODUCER) - params = { - "do_remote_decode": True, - "transfer_id": "tx", - "remote_engine_id": "decode", - "remote_block_ids": [], - "remote_host": "127.0.0.1", - "remote_handshake_port": 6000, - "remote_notify_port": 6001, - } - req = _mk_request(list(range(9)), params=params) - req.request_id = "req" - req.num_computed_tokens = 0 - sched._reqs_need_save["req"] = (req, [[10, 11, 12], [1]]) - sched._req_kv_params["req"] = dict(params) - - def _output(cached_req_ids, new_block_ids, num_scheduled): - return SimpleNamespace( - scheduled_cached_reqs=SimpleNamespace( - req_ids=cached_req_ids, new_block_ids=new_block_ids - ), - num_scheduled_tokens={"req": num_scheduled}, - new_block_ids_to_zero=None, - kv_connector_block_state=KVConnectorBlockState( - req_ids={"req"}, - resolve_block_ids=lambda _: ([0, 51, 52, 53], [31, 32, 33]), - boundary_state_offloads={}, - ), - ) - - # First chunk: not final, so nothing is saved yet. - meta = sched.build_connector_meta(_output([], [], 4)) - assert not meta.reqs_to_save - assert "req" in sched._reqs_need_pending_save - - # Final chunk: stale cached ids plus new ids are ignored in favour of - # the scheduler's current table, clamped to the prompt. - req.num_computed_tokens = 4 - meta = sched.build_connector_meta(_output(["req"], [([77], [78])], 5)) - assert meta.reqs_to_save["req"].local_block_ids == [[51], [31, 32]] - assert "req" not in sched._reqs_need_pending_save - - -@pytest.mark.parametrize( - "mode,deadlines,expected", - [ - (MoRIIOMode.WRITE, {"req": (0.0, "tx")}, True), - (MoRIIOMode.WRITE, {}, False), - (MoRIIOMode.READ, {"req": (0.0, "tx")}, False), - ], -) -def test_has_pending_push_work_tracks_write_deferred_frees(mode, deadlines, expected): - connector = _FakeConnector(_FakeScheduler(_deferred_send_deadlines=deadlines)) - connector.mode = mode - assert connector.has_pending_push_work() is expected - - -def test_has_pending_push_work_is_false_on_worker_role(): - connector = _FakeConnector(None) - connector.mode = MoRIIOMode.WRITE - assert connector.has_pending_push_work() is False - - -def test_loaded_kv_cache_groups_are_every_transfer_group(): - # The connector restores the request's own blocks, not a hash-addressed - # prefix, so a transfer-enabled Mamba group counts as loaded too. - connector = _FakeConnector(None) - connector._kv_cache_config = SimpleNamespace( - transfer_group_ids=(0, 2), prefix_cacheable_group_ids=(0,) - ) - assert connector.get_loaded_kv_cache_group_ids(SimpleNamespace()) == (0, 2) - - -def test_split_block_groups_rejects_non_hybrid_use(): - sched = _FakeScheduler(_has_mamba=False) - - with pytest.raises(moriio_common.MoRIIOError, match="non-hybrid"): - sched.split_block_groups(([5, 6, 7],)) - - -def test_split_block_groups_accepts_empty_abort_payload(): - sched = _FakeScheduler(_has_mamba=True) - - assert sched.split_block_groups(()) == ([], []) - - -def test_split_block_groups_keeps_only_running_state(): - sched = _FakeScheduler( - _has_mamba=True, - _attn_group_ids=[0], - _mamba_group_ids=[1], - ) - assert sched.split_block_groups(([1], [40, 41])) == ([1], [[41]]) - - -@pytest.mark.parametrize( - "blocks,expected", - [ - # The running state is picked only after the scratch slots are gone. - ([40, 41, 42, 43], [41]), - # Never empty: a list shorter than the scratch count keeps one state. - ([40], [40]), - ([], []), - ], -) -def test_split_block_groups_strips_dspark_scratch_slots(blocks, expected): - sched = _FakeScheduler( - _has_mamba=True, - _attn_group_ids=[0], - _mamba_group_ids=[1], - _num_ssm_scratch_blocks=2, - ) - - assert sched.split_block_groups(([1], blocks)) == ([1], [expected]) - - -# -------------------------------------------------------------------------- -# request_finished_all_groups -# -------------------------------------------------------------------------- -def test_request_finished_all_groups_carries_attn_and_mamba_in_one_field(): - seen = {} - - def _fake_request_finished(request, attn_block_ids, mamba_block_groups): - seen["attn"] = list(attn_block_ids) - seen["mamba"] = [list(group) for group in mamba_block_groups] - return True, { - "do_remote_prefill": True, - "remote_block_ids": [attn_block_ids, *mamba_block_groups], - } - - sched = _FakeScheduler(_has_mamba=True, _attn_group_ids=[0], _mamba_group_ids=[1]) - sched.request_finished = _fake_request_finished - conn = _FakeConnector(sched) - - request = SimpleNamespace(request_id="r0") - block_ids = ([10, 11], [42]) - delay_free, params = conn.request_finished_all_groups(request, block_ids) - - assert delay_free is True - # Attention ids drive request_finished; the mamba slot rides the SAME - # remote_block_ids channel as [attn, *mamba_groups], so the proxy/router - # needs no KDA-specific field. - assert seen["attn"] == [10, 11] - assert seen["mamba"] == [[42]] - assert params["remote_block_ids"] == [[10, 11], [42]] - assert "remote_mamba_block_ids" not in params - - -def test_request_finished_all_groups_pure_attention_stays_grouped(): - def _fake_request_finished(request, block_ids): - return True, {"remote_block_ids": block_ids} - - sched = _FakeScheduler(_has_mamba=False, _mamba_group_ids=[]) - sched.request_finished = _fake_request_finished - conn = _FakeConnector(sched) - - delay_free, params = conn.request_finished_all_groups( - SimpleNamespace(request_id="r1"), ([7, 8],) - ) - # Main's HMA contract keeps pure-attention block ids grouped. - assert params["remote_block_ids"] == ([7, 8],) - assert "remote_mamba_block_ids" not in params - - -def test_connector_supports_divergent_local_hybrid_hits(): - connector = _FakeConnector(None) - connector._transfers_mamba_state = True - connector.mode = MoRIIOMode.READ - assert connector.supports_divergent_local_hybrid_hits is True - connector.mode = MoRIIOMode.WRITE - assert connector.supports_divergent_local_hybrid_hits is False - - connector.mode = MoRIIOMode.READ - connector._transfers_mamba_state = False - assert connector.supports_divergent_local_hybrid_hits is False - - -def _make_read_scheduler(): - return _FakeScheduler( - is_producer=False, - mode=MoRIIOMode.READ, - _has_mamba=True, - _attn_group_ids=[0], - _mamba_group_ids=[1], - request_id_to_transfer_id={}, - transfer_id_to_request_id={}, - _reqs_need_recv={}, - _reqs_need_save={}, - _reqs_need_send={}, - _req_kv_params={}, - _max_decode_tail_blocks=1, - ) - - -def _make_read_request(remote_block_ids): - return SimpleNamespace( - request_id="req", - kv_transfer_params={ - "do_remote_prefill": True, - "transfer_id": "tx", - "remote_engine_id": "prefill", - "remote_block_ids": remote_block_ids, - "remote_host": "127.0.0.1", - "remote_handshake_port": 6000, - "remote_notify_port": 6001, - }, - ) - - -@pytest.mark.parametrize("external_tokens", [0, 256]) -@pytest.mark.parametrize("has_mamba", [False, True]) -@pytest.mark.parametrize("zero_ids", [None, [], [101], [101, 102, 202, 999]]) -def test_update_state_drops_decode_recompute_tail_block( - external_tokens, has_mamba, zero_ids -): - """Only full hybrid READ destinations skip zeroing; local tails still zero.""" - sched = _make_read_scheduler() - sched._has_mamba = has_mamba - remote_blocks = [[10, 11], [91]] if has_mamba else [[10, 11]] - request = _make_read_request(remote_blocks) - blocks = _FakeBlocks( - all_groups=([100, 101, 102], [200, 201, 202]) - if has_mamba - else ([100, 101, 102],), - ) - - sched.update_state_after_alloc(request, blocks, num_external_tokens=external_tokens) - - expected_blocks = [[100, 101] if external_tokens else []] - if has_mamba: - expected_blocks.append([202]) - assert sched._reqs_need_recv["req"][1] == expected_blocks - assert sched._req_kv_params["req"]["remote_block_ids"] == remote_blocks - - output = SchedulerOutput.make_empty() - output.new_block_ids_to_zero = None if zero_ids is None else list(zero_ids) - before = asdict(output) - meta = sched.build_connector_meta(output) - assert meta.reqs_to_recv["req"].local_block_ids == expected_blocks - if zero_ids and has_mamba and external_tokens: - before["new_block_ids_to_zero"] = zero_ids[1:] - assert asdict(output) == before - - # A later local allocation of the same pages must still be zeroed. - output.new_block_ids_to_zero = [100, 101, 102] - meta = sched.build_connector_meta(output) - assert not meta.reqs_to_recv - assert output.new_block_ids_to_zero == [100, 101, 102] - - -def test_update_state_pairs_shorter_local_blocks_with_remote_suffix(): - sched = _make_read_scheduler() - request = _make_read_request([[10, 11, 12], [92]]) - blocks = _FakeBlocks( - all_groups=([102], [202]), - ) - - sched.update_state_after_alloc(request, blocks, num_external_tokens=64) - - assert sched._reqs_need_recv["req"][1] == [[102], [202]] - assert sched._req_kv_params["req"]["remote_block_ids"] == [[12], [92]] - - -def test_update_state_pairs_trimmed_mamba_state_with_remote_state(): - """The DSpark shape end to end: scratch slots are stripped, the running - state is then selected, and the attention list loses its lookahead tail.""" - sched = _make_read_scheduler() - sched._num_ssm_scratch_blocks = 2 - sched._max_decode_tail_blocks = 2 - request = _make_read_request([[10, 11], [90]]) - # Mamba group: [null, running state, scratch, scratch]. - blocks = _FakeBlocks(all_groups=([100, 101, 102, 103], [200, 201, 202, 203])) - - sched.update_state_after_alloc(request, blocks, num_external_tokens=256) - - assert sched._reqs_need_recv["req"][1] == [[100, 101], [201]] - assert sched._req_kv_params["req"]["remote_block_ids"] == [[10, 11], [90]] - - -def test_update_state_full_attention_hit_still_carries_mamba_state(): - sched = _make_read_scheduler() - request = _make_read_request([[10], [90]]) - - sched.update_state_after_alloc( - request, - _FakeBlocks(all_groups=([100], [200])), - num_external_tokens=0, - ) - - assert sched._reqs_need_recv["req"][1] == [[], [200]] - - -def test_update_state_preserves_each_mamba_group(): - sched = _make_read_scheduler() - sched._mamba_group_ids = [1, 2] - request = _make_read_request([[10], [90], [80]]) - - sched.update_state_after_alloc( - request, - _FakeBlocks(all_groups=([100], [200], [300])), - num_external_tokens=64, - ) - - assert sched._reqs_need_recv["req"][1] == [[100], [200], [300]] - assert sched._req_kv_params["req"]["remote_block_ids"] == [ - [10], - [90], - [80], - ] - - -def test_align_read_blocks_allows_configured_lookahead_tail(): - assert moriio_connector.MoRIIOConnectorScheduler._align_read_blocks( - [100, 101, 102], [10], max_decode_tail_blocks=2 - ) == ([100], [10]) - with pytest.raises(ValueError, match="allowed local tail"): - moriio_connector.MoRIIOConnectorScheduler._align_read_blocks( - [100, 101, 102], [10], max_decode_tail_blocks=1 - ) - - -def test_update_state_aligns_attention_only_decode_tail(): - sched = _make_read_scheduler() - sched._has_mamba = False - sched._mamba_group_ids = [] - request = _make_read_request([[10, 11]]) - - sched.update_state_after_alloc( - request, - _FakeBlocks(all_groups=([100, 101, 102],)), - num_external_tokens=256, - ) - - assert sched._reqs_need_recv["req"][1] == [[100, 101]] - assert sched._req_kv_params["req"]["remote_block_ids"] == [[10, 11]] - - -def test_read_producer_does_not_queue_write_only_save_state(): - sched = _FakeScheduler( - is_producer=True, - mode=MoRIIOMode.READ, - _has_mamba=True, - _attn_group_ids=[0], - _mamba_group_ids=[1], - request_id_to_transfer_id={}, - transfer_id_to_request_id={}, - _reqs_need_save={}, - _req_kv_params={}, - ) - request = SimpleNamespace( - request_id="req", - kv_transfer_params={"do_remote_decode": True, "transfer_id": "tx"}, - ) - - sched.update_state_after_alloc( - request, - _FakeBlocks(all_groups=([100, 101], [200])), - num_external_tokens=0, - ) - - assert sched._reqs_need_save == {} - assert sched._req_kv_params == {} - - -def test_write_producer_queues_grouped_attention_blocks(): - sched = _FakeScheduler( - is_producer=True, - mode=MoRIIOMode.WRITE, - _has_mamba=False, - _attn_group_ids=[0], - _mamba_group_ids=[], - request_id_to_transfer_id={}, - transfer_id_to_request_id={}, - _reqs_need_save={}, - _req_kv_params={}, - ) - request = SimpleNamespace( - request_id="req", - kv_transfer_params={"do_remote_decode": True, "transfer_id": "tx"}, - ) - - sched.update_state_after_alloc( - request, - _FakeBlocks(all_groups=([100, 101],)), - num_external_tokens=0, - ) - - assert sched._reqs_need_save["req"][1] == ([100, 101],) - assert sched._req_kv_params["req"] == request.kv_transfer_params - - -def test_session_build_rejects_per_layer_region_count_mismatch(): - worker = _FakeWorker( - built_write_session={}, - layer_name_to_local_kv_cache_metadata={"kda.0": ["conv", "ssm"]}, - layer_name_to_remote_kv_cache_metadata={"prefill": {"kda.0": ["conv"]}}, - ) - - with pytest.raises(moriio_common.MoRIIOError, match="registered 1 region"): - worker._get_built_session("prefill") - - -def test_register_kv_caches_rejects_invalid_hybrid_write_state_layout(): - worker = _FakeWorker( - mode=MoRIIOMode.WRITE, - _transfer_layer_names={"kda.0"}, - layer_to_spec={"kda.0": _mamba_spec(1)}, - ) - worker._is_mamba_layer = lambda _layer_name: True - - with pytest.raises(moriio_common.MoRIIOError, match="exactly two Mamba states"): - worker.register_kv_caches({"kda.0": object()}) - - -def test_register_kv_caches_rejects_no_transfer_enabled_layers(): - worker = _FakeWorker(mode=MoRIIOMode.READ, _transfer_layer_names=set()) - - with pytest.raises(moriio_common.MoRIIOError, match="no transfer-enabled"): - worker.register_kv_caches({"disabled.0": object()}) - - -def test_worker_rejects_incompatible_mamba_specs_before_registration(): - incompatible = _mamba_spec() - object.__setattr__(incompatible, "page_size_padded", 4096) - worker = _FakeWorker( - mode=MoRIIOMode.READ, - world_size=1, - _transfer_layer_names={"kda.0", "kda.1"}, - layer_to_spec={"kda.0": _mamba_spec(), "kda.1": incompatible}, - ) - - with pytest.raises(moriio_common.MoRIIOError, match="same cache spec"): - worker.register_kv_caches({"kda.0": object(), "kda.1": object()}) - - -def _make_register_worker(caches, geoms, *, disable_hma: bool = True): - worker = _FakeWorker( - mode=MoRIIOMode.READ, - _transfer_layer_names=set(caches), - block_size=16, - engine_id="e0", - dst_num_blocks={}, - block_lens={}, - layer_name_to_local_kv_cache_metadata={}, - kv_caches_base_addr={}, - kv_layer_mr_offset={}, - local_kv_cache_size=[], - block_window_per_layer=[], - moriio_wrapper=SimpleNamespace( - register_local_tensor=lambda _t: b"m", - get_agent_metadata=lambda: b"a", - async_wait_reqid=lambda: None, - ), - world_size=1, - layer_to_spec={}, - backend_name="TRITON_ATTN", - side_channel_port=6301, - tp_rank=0, - dp_rank=0, - is_producer=False, - vllm_config=SimpleNamespace( - model_config=SimpleNamespace(hf_config=SimpleNamespace(model_type="x")), - scheduler_config=SimpleNamespace( - disable_hybrid_kv_cache_manager=disable_hma - ), - ), - ) - worker._is_mamba_layer = lambda _n: False - worker._is_mla_cache_layer = lambda _n: False - worker._get_layer_transfer_geometry = lambda n, remote_num_blocks=None: geoms[n] - worker._iter_layer_registration_regions = lambda n: [(caches[n], caches[n].nbytes)] - worker._moriio_handshake_listener = ( - lambda _m, ready_event, *_a, **_k: ready_event.set() - ) - return worker - - -def test_register_kv_caches_allows_hybrid_divergent_block_len(): - """GLM-5.2 / MiniMax HMA: divergent per-layer block_len is allowed.""" - Geom = moriio_layout.LayerTransferGeometry - caches = { - "layer0": torch.zeros(2, 4, 16, 1, 8, dtype=torch.float16), - "layer1": torch.zeros(2, 4, 16, 1, 8, dtype=torch.float16), - } - # Same tokens-per-block (16); different bytes-per-block (2112 vs 9216). - geoms = { - "layer0": Geom(4, 16, 2112, 132, 2112, None, None, 1, 1, False), - "layer1": Geom(4, 16, 9216, 576, 9216, None, None, 1, 1, False), - } - worker = _make_register_worker(caches, geoms) - worker.register_kv_caches(caches) - - assert worker.block_lens == {"layer0": 2112, "layer1": 9216} - assert worker.block_len == 9216 - - -def test_register_kv_caches_rejects_divergent_block_size(): - """Without HMA, block_size (tokens/page) must stay uniform.""" - Geom = moriio_layout.LayerTransferGeometry - caches = { - "layer0": torch.zeros(2, 4, 16, 1, 8, dtype=torch.float16), - "layer1": torch.zeros(2, 4, 16, 1, 8, dtype=torch.float16), - } - geoms = { - "layer0": Geom(4, 16, 2112, 132, 2112, None, None, 1, 1, False), - "layer1": Geom(4, 32, 2112, 66, 2112, None, None, 1, 1, False), - } - worker = _make_register_worker(caches, geoms, disable_hma=True) - with pytest.raises(ValueError, match="block size mismatch"): - worker.register_kv_caches(caches) - - -def test_register_kv_caches_rejects_divergent_block_size_even_if_block_len_differs(): - """Without HMA, size+len both differing must still raise.""" - Geom = moriio_layout.LayerTransferGeometry - caches = { - "layer0": torch.zeros(2, 4, 16, 1, 8, dtype=torch.float16), - "layer1": torch.zeros(2, 4, 16, 1, 8, dtype=torch.float16), - } - # Previously buggy: size+len both differ → registration continued. - geoms = { - "layer0": Geom(4, 16, 2112, 132, 2112, None, None, 1, 1, False), - "layer1": Geom(4, 32, 9216, 288, 9216, None, None, 1, 1, False), - } - worker = _make_register_worker(caches, geoms, disable_hma=True) - with pytest.raises(ValueError, match="block size mismatch"): - worker.register_kv_caches(caches) - - -def test_register_kv_caches_allows_hma_divergent_block_size(): - """Gemma4 HMA (#53721): SWA vs full-attn may differ in tokens/page.""" - Geom = moriio_layout.LayerTransferGeometry - caches = { - "layer0": torch.zeros(2, 4, 16, 1, 8, dtype=torch.float16), - "layer1": torch.zeros(2, 4, 16, 1, 8, dtype=torch.float16), - } - geoms = { - "layer0": Geom(4, 16, 2112, 132, 2112, None, None, 1, 1, False), - "layer1": Geom(4, 32, 4224, 132, 4224, None, None, 1, 1, False), - } - worker = _make_register_worker(caches, geoms, disable_hma=False) - worker.register_kv_caches(caches) - assert worker.block_lens == {"layer0": 2112, "layer1": 4224} - assert worker.block_len == 4224 - - -# -------------------------------------------------------------------------- -# _truncate_mamba_request_for_prefill -# -------------------------------------------------------------------------- -def _mk_request(prompt, max_tokens=64, params=None): - return SimpleNamespace( - kv_transfer_params={} if params is None else params, - num_prompt_tokens=len(prompt), - prompt_token_ids=list(prompt), - prompt_embeds=None, - _all_token_ids=list(prompt), - max_tokens=max_tokens, - ) - - -def test_truncate_mamba_request_pops_last_token_and_caps_tokens(): - sched = _FakeScheduler() - req = _mk_request([10, 11, 12, 13, 14]) - sched._truncate_mamba_request_for_prefill(req) - - assert req.prompt_token_ids == [10, 11, 12, 13] # last token popped - assert req._all_token_ids == [10, 11, 12, 13] - assert req.num_prompt_tokens == 4 # N-1 accounting - assert req.max_tokens == 1 - assert req.kv_transfer_params["_p_side_truncated"] is True - - -def test_truncate_mamba_request_is_idempotent_across_reschedule(): - sched = _FakeScheduler() - req = _mk_request([10, 11, 12]) - sched._truncate_mamba_request_for_prefill(req) - assert req.num_prompt_tokens == 2 - # A second call (e.g. after a preemption) must not truncate again. - sched._truncate_mamba_request_for_prefill(req) - assert req.num_prompt_tokens == 2 - assert req.prompt_token_ids == [10, 11] - - -def test_truncate_mamba_request_noop_for_single_token_prompt(): - sched = _FakeScheduler() - req = _mk_request([10]) - sched._truncate_mamba_request_for_prefill(req) - assert req.num_prompt_tokens == 1 - assert req.prompt_token_ids == [10] - assert "_p_side_truncated" not in req.kv_transfer_params - - -# -------------------------------------------------------------------------- -# get_num_new_matched_tokens (hybrid N-1 accounting) -# -------------------------------------------------------------------------- -@pytest.mark.parametrize( - "params,expected", - [ - (None, (0, False)), - ({}, (0, False)), - ({"do_remote_prefill": False}, (0, False)), - ({"do_remote_prefill": True}, (31, False)), - ], -) -def test_read_matches_only_requests_with_pending_remote_prefill(params, expected): - scheduler = _FakeScheduler(is_producer=False, mode=MoRIIOMode.READ) - request = SimpleNamespace(num_prompt_tokens=32, kv_transfer_params=params) - - assert scheduler.get_num_new_matched_tokens(request, 0) == expected - - -@pytest.mark.parametrize( - "has_mamba,expected", [(False, (32, True)), (True, (31, True))] -) -def test_write_consumer_matches_full_prompt_unless_hybrid(has_mamba, expected): - """Without a mamba group (attention-only, or HMA disabled) prefill pushed - KV for all N tokens; with one it pushed h(N-1).""" - scheduler = _FakeScheduler( - is_producer=False, mode=MoRIIOMode.WRITE, _has_mamba=has_mamba - ) - request = SimpleNamespace( - num_prompt_tokens=32, kv_transfer_params={"do_remote_prefill": True} - ) - - assert scheduler.get_num_new_matched_tokens(request, 0) == expected - - -@pytest.mark.parametrize( - "is_producer,has_mamba,expected_prompt_len", - [(True, True, 4), (True, False, 5), (False, True, 5)], -) -def test_on_new_request_truncates_only_hybrid_producers( - is_producer, has_mamba, expected_prompt_len -): - """Attention-only prefill (including models run with HMA disabled) - computes all N prompt tokens; only a hybrid producer drops the last.""" - scheduler = _FakeScheduler(is_producer=is_producer, _has_mamba=has_mamba) - request = _mk_request([10, 11, 12, 13, 14], params={"do_remote_decode": True}) - - scheduler.on_new_request(request) - - assert request.num_prompt_tokens == expected_prompt_len - assert len(request.prompt_token_ids) == expected_prompt_len - - -def test_read_rematch_after_alloc_keeps_queued_transfer(): - """Clearing the flag blocks another external hit, not the queued READ.""" - sched = _make_read_scheduler() - request = _make_read_request([[10, 11], [91]]) - request.num_prompt_tokens = 32 - assert sched.get_num_new_matched_tokens(request, 0) == (31, False) - - sched.update_state_after_alloc( - request, _FakeBlocks(all_groups=([100, 101], [200])), num_external_tokens=31 - ) - - assert sched._reqs_need_recv["req"][1] == [[100, 101], [200]] - assert request.kv_transfer_params["do_remote_prefill"] is False - - meta = sched.build_connector_meta(SchedulerOutput.make_empty()) - queued = meta.reqs_to_recv["req"] - assert queued.transfer_id == "tx" - assert queued.local_block_ids == [[100, 101], [200]] - assert queued.remote_block_ids == [[10, 11], [91]] - assert sched.get_num_new_matched_tokens(request, 0) == (0, False) - - -def test_get_num_new_matched_tokens_read_recomputes_last_token(): - sched = _FakeScheduler(is_producer=False, mode=MoRIIOMode.READ, _has_mamba=True) - req = SimpleNamespace( - num_prompt_tokens=10, - prompt_token_ids=list(range(10)), - kv_transfer_params={"do_remote_prefill": True}, - ) - n, is_async = sched.get_num_new_matched_tokens(req, num_computed_tokens=0) - # READ always recomputes the final token locally: N-1 - computed. - assert n == 9 - assert is_async is False - - -def test_get_num_new_matched_tokens_write_plain_keeps_all_tokens(): - sched = _FakeScheduler(is_producer=False, mode=MoRIIOMode.WRITE, _has_mamba=False) - req = SimpleNamespace( - num_prompt_tokens=10, - prompt_token_ids=list(range(10)), - kv_transfer_params={"do_remote_prefill": True}, - ) - n, is_async = sched.get_num_new_matched_tokens(req, num_computed_tokens=2) - # Pure-attention WRITE: no N-1 drop; full length minus already-computed. - assert n == 8 - assert is_async is True - - -@pytest.mark.parametrize("params", [None, {}, {"do_remote_prefill": False}]) -def test_write_does_not_wait_again_after_receive_or_for_local_requests(params): - sched = _FakeScheduler(is_producer=False, mode=MoRIIOMode.WRITE, _has_mamba=True) - req = _mk_request(list(range(10))) - req.kv_transfer_params = params - assert sched.get_num_new_matched_tokens(req, 0) == (0, False) - - -def test_write_never_declares_an_empty_async_load(): - sched = _FakeScheduler(is_producer=False, mode=MoRIIOMode.WRITE, _has_mamba=False) - req = _mk_request(list(range(10)), params={"do_remote_prefill": True}) - assert sched.get_num_new_matched_tokens(req, 10) == (0, False) - - -@pytest.mark.parametrize( - "prompt_len,computed,expected", [(16, 0, 15), (17, 16, 0), (1, 0, 0)] -) -def test_hybrid_write_advertises_only_the_state_computed_by_prefill( - prompt_len, computed, expected -): - sched = _FakeScheduler(is_producer=False, mode=MoRIIOMode.WRITE, _has_mamba=True) - req = _mk_request(list(range(prompt_len)), params={"do_remote_prefill": True}) - assert sched.get_num_new_matched_tokens(req, computed) == (expected, expected > 0) - - -@pytest.mark.parametrize( - ("mode", "num_computed_tokens", "expected", "is_async"), - [ - (MoRIIOMode.READ, 0, 9, False), - (MoRIIOMode.READ, 10, 0, False), - (MoRIIOMode.WRITE, 2, 7, True), - ], -) -def test_get_num_new_matched_tokens_supports_embeds_only_prompts( - mode, num_computed_tokens, expected, is_async -): - sched = _FakeScheduler(is_producer=False, mode=mode, _has_mamba=True) - req = SimpleNamespace( - num_prompt_tokens=10, - prompt_token_ids=None, - prompt_embeds=object(), - kv_transfer_params={"do_remote_prefill": True}, - ) - - assert sched.get_num_new_matched_tokens(req, num_computed_tokens) == ( - expected, - is_async, - ) - - -def test_get_num_new_matched_tokens_producer_returns_zero(): - sched = _FakeScheduler(is_producer=True, mode=MoRIIOMode.WRITE, _has_mamba=True) - req = _mk_request(list(range(5)), params={"do_remote_decode": True}) - n, is_async = sched.get_num_new_matched_tokens(req, num_computed_tokens=0) - assert (n, is_async) == (0, False) - # The producer never truncates here: the scheduler has already measured the - # prefix-cache hit against the full prompt by this point. - assert req.num_prompt_tokens == 5 - - -def test_on_new_request_truncates_producer_prompt(): - sched = _FakeScheduler(is_producer=True, mode=MoRIIOMode.WRITE, _has_mamba=True) - req = _mk_request(list(range(5)), params={"do_remote_decode": True}) - sched.on_new_request(req) - # Producer stops at h(N-1): the last prompt token was dropped. - assert req.num_prompt_tokens == 4 - assert req.max_tokens == 1 - assert req.kv_transfer_params["_p_side_truncated"] is True - - -def test_on_new_request_is_noop_without_mamba_or_on_decode(): - plain = _FakeScheduler(is_producer=True, mode=MoRIIOMode.WRITE, _has_mamba=False) - req = _mk_request(list(range(5)), params={"do_remote_decode": True}) - plain.on_new_request(req) - assert req.num_prompt_tokens == 5 - - decode_side = _FakeScheduler( - is_producer=False, mode=MoRIIOMode.WRITE, _has_mamba=True - ) - req2 = _mk_request(list(range(5)), params={"do_remote_prefill": True}) - decode_side.on_new_request(req2) - assert req2.num_prompt_tokens == 5 - - -# -------------------------------------------------------------------------- -# Offset-template cache -# -------------------------------------------------------------------------- -_SAMPLE_SLOT_SETS = [ - ([1, 2], [1, 2]), - ([0], [3]), - ([3, 0, 2], [3, 0, 2]), - ([2, 2], [5, 5]), - ([], []), -] - - -def test_worker_compute_mamba_offsets_caches_and_matches_template(): - split = _gdn_split_info() - conv_dim = sum(split.local_proj_dims) - conv_shape = (conv_dim, split.conv_rows) - ssm_shape = (2, 4, 4) - page_bytes = 256 - pages = torch.zeros((8, 1, 1, page_bytes), dtype=torch.uint8) - spec = moriio_connector.MambaSpec( - block_size=1, - shapes=(conv_shape, ssm_shape), - dtypes=(torch.bfloat16, torch.bfloat16), - page_size_padded=page_bytes, - ) - conv, ssm = moriio_layout.kda_conv_ssm(pages, spec) - - worker = _FakeWorker( - kv_caches={"kda.0": pages}, - layer_to_spec={"kda.0": spec}, - _conv_decomp=split, - _mamba_offset_templates={}, - tp_rank=0, - world_size=1, - ) - - cached_template = None - for local_slots, remote_slots in _SAMPLE_SLOT_SETS: - lo, ro, sz, n_conv = worker._compute_mamba_transfer_offsets( - "kda.0", local_slots, remote_slots - ) - template = moriio_layout.build_mamba_offset_template( - conv, ssm, split, tp_ratio=1 - ) - expected = moriio_layout.apply_mamba_offset_template( - template, local_slots, remote_slots - ) - assert (lo, ro, sz) == expected - assert n_conv == moriio_layout.compute_mamba_conv_split_count( - local_slots, split - ) - - current_template = worker._mamba_offset_templates["kda.0"] - if cached_template is None: - cached_template = current_template - else: - assert current_template is cached_template - - assert "kda.0" in worker._mamba_offset_templates - - -@pytest.mark.parametrize("remote_tp_size", [1, None]) -def test_mamba_tp_ratio_accepts_only_equal_tp(remote_tp_size): - worker = _FakeWorker(world_size=1) - assert worker._mamba_tp_ratio(remote_tp_size) == 1 - - -@pytest.mark.parametrize("remote_tp_size", [2, 3]) -def test_mamba_tp_ratio_rejects_heterogeneous_tp(remote_tp_size): - worker = _FakeWorker(world_size=1) - with pytest.raises(NotImplementedError, match="heterogeneous-TP"): - worker._mamba_tp_ratio(remote_tp_size) - - -def test_worker_selects_each_mamba_layers_own_block_group(): - worker = _FakeWorker(_mamba_payload_index_by_layer={"kda.0": 1, "kda.1": 2}) - groups = ([10], [40], [70]) - - assert worker._mamba_blocks_for_layer("kda.0", groups) == [40] - assert worker._mamba_blocks_for_layer("kda.1", groups) == [70] - - -def test_worker_rejects_missing_mamba_block_group(): - worker = _FakeWorker(_mamba_payload_index_by_layer={"kda.0": 2}) - - with pytest.raises(moriio_common.MoRIIOError, match="no block group"): - worker._mamba_blocks_for_layer("kda.0", ([10], [40])) diff --git a/tests/v1/kv_connector/unit/test_moriio_proxy_routing.py b/tests/v1/kv_connector/unit/test_moriio_proxy_routing.py deleted file mode 100644 index e3e04aedb8..0000000000 --- a/tests/v1/kv_connector/unit/test_moriio_proxy_routing.py +++ /dev/null @@ -1,335 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Hardware-fair request routing in the MoRIIO toy P/D proxy. - -Exercises the REAL ``flat_interleaved_dp_route`` from the toy proxy (loaded from -its ``examples/`` path with heavy deps stubbed) — no routing logic is copied -here. This is the request-distribution half of end-to-end fairness: the proxy -must hand every prefill/decode (instance, dp_rank) slot an equal share so no GPU -is starved. The connector-side read routing is covered in -``test_moriio_routing_fairness.py``. - -Regression target: the previous scheme derived ``instance = req % n_instances`` -and ``dp_rank = req % dp_size`` from the same counter, so when -``n_instances | dp_size`` each instance was locked to a stride-``n`` subset of -its ranks — e.g. 2 prefill instances x DP8 stranded 4 of every node's 8 GPUs. -``flat_interleaved_dp_route`` walks ONE counter over the full -``(instance, dp_rank)`` slot space, so the two selections can never alias. - -Role shapes below are exactly those in the RFC (#46107) deployments: - (1, 1) 1P/1D TP8 (2, 1) 2P/2D TP8 (4, 1) 4D TP8 - (1, 8) 1D DP8EP (2, 8) 2P DP8EP (3, 8) 3D DP8EP -""" - -import contextlib -import ctypes -import importlib.util -import os -import subprocess -import sys -import time -import types -from collections import Counter -from pathlib import Path -from typing import cast - -import msgpack -import pytest -import zmq - -_MISSING = object() - -PROXY_REL = "examples/disaggregated/disaggregated_serving/moriio_toy_proxy_server.py" - - -def _module(name, **attrs): - module = types.ModuleType(name) - for attr, value in attrs.items(): - setattr(module, attr, value) - return module - - -def _package(name): - module = _module(name) - module.__path__ = [] - return module - - -class _QuartStub: - def __init__(self, *a, **k): - pass - - def route(self, *a, **k): - return lambda fn: fn - - def post(self, *a, **k): - return self.route() - - -async def _make_response_stub(value): - return value - - -@contextlib.contextmanager -def _proxy_import_stubs(): - """Stub the proxy's external deps so the module imports for a pure unit test. - - Only third-party/vllm imports are stubbed; the routing function under test - is executed as-is from the real module. - """ - common = "vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_common" - - class _MoRIIOConstants: - TRANSFER_PREFIX = "moriio-transfer" - - stubs = { - "aiohttp": _module("aiohttp"), - "msgpack": _module("msgpack"), - "zmq": _module("zmq"), - "quart": _module( - "quart", - Quart=_QuartStub, - Request=object, - make_response=_make_response_stub, - request=object(), - ), - "vllm": _package("vllm"), - "vllm.distributed": _package("vllm.distributed"), - "vllm.distributed.kv_transfer": _package("vllm.distributed.kv_transfer"), - "vllm.distributed.kv_transfer.kv_connector": _package( - "vllm.distributed.kv_transfer.kv_connector" - ), - "vllm.distributed.kv_transfer.kv_connector.v1": _package( - "vllm.distributed.kv_transfer.kv_connector.v1" - ), - "vllm.distributed.kv_transfer.kv_connector.v1.moriio": _package( - "vllm.distributed.kv_transfer.kv_connector.v1.moriio" - ), - common: _module(common, MoRIIOConstants=_MoRIIOConstants), - } - saved = {} - for name, module in stubs.items(): - if name not in sys.modules: - saved[name] = _MISSING - sys.modules[name] = module - try: - yield - finally: - for name, previous in saved.items(): - if previous is _MISSING: - sys.modules.pop(name, None) - else: - sys.modules[name] = cast(types.ModuleType, previous) - - -def _load_proxy_module(): - path = Path(__file__).parents[4] / PROXY_REL - spec = importlib.util.spec_from_file_location("moriio_proxy_under_test", path) - assert spec is not None and spec.loader is not None - module = importlib.util.module_from_spec(spec) - with _proxy_import_stubs(): - spec.loader.exec_module(module) - return module - - -@pytest.fixture(scope="module") -def route(): - return _load_proxy_module().flat_interleaved_dp_route - - -def _instances(n: int, dp_size: int): - # Only dp_size is read by the router; tp_size carried for realism. - return [{"dp_size": dp_size, "tp_size": 8 // dp_size} for _ in range(n)] - - -# (n_instances, dp_size) for every distinct P/D role shape in the RFC configs. -ROLE_SHAPES = [(1, 1), (2, 1), (4, 1), (1, 8), (2, 8), (3, 8)] -SHAPE_IDS = [f"n{n}_dp{dp}" for n, dp in ROLE_SHAPES] - - -def _route_n(route, instances, count): - # Proxy uses 1-indexed request numbers (slot = (request_number - 1) % ...). - return [route(rn, instances) for rn in range(1, count + 1)] - - -@pytest.mark.parametrize(("n", "dp"), ROLE_SHAPES, ids=SHAPE_IDS) -def test_full_slot_space_is_covered_uniformly(route, n, dp): - instances = _instances(n, dp) - period = n * dp - # Three full cycles -> every (instance, dp_rank) slot must be hit the same - # number of times (exactly uniform, no starved slot, no aliasing). - hits = Counter(_route_n(route, instances, period * 3)) - - expected: set[tuple[int, int | None]] - if dp == 1: - expected = {(inst, None) for inst in range(n)} - else: - expected = {(inst, r) for inst in range(n) for r in range(dp)} - assert set(hits) == expected, f"missing slots: {expected - set(hits)}" - assert max(hits.values()) == min(hits.values()) - - -@pytest.mark.parametrize(("n", "dp"), ROLE_SHAPES, ids=SHAPE_IDS) -def test_instance_and_dp_rank_marginals_are_balanced(route, n, dp): - instances = _instances(n, dp) - routed = _route_n(route, instances, n * dp * 5) - - inst_counts = Counter(inst for inst, _ in routed) - assert set(inst_counts) == set(range(n)) - assert max(inst_counts.values()) == min(inst_counts.values()) - - dp_counts = Counter(r for _, r in routed) - if dp == 1: - assert set(dp_counts) == {None} - else: - assert set(dp_counts) == set(range(dp)) - assert max(dp_counts.values()) == min(dp_counts.values()) - - -def test_tp_instance_forwards_no_dp_rank(route): - # dp_size == 1 (a TP instance) must yield dp_rank None so the proxy never - # forwards an out-of-range data-parallel rank. - assert all(r is None for _, r in _route_n(route, _instances(2, 1), 8)) - - -def test_two_instance_dp8_gives_every_node_all_ranks(route): - # Direct regression for the stranded-GPU bug: with 2 prefill instances x DP8 - # each instance must receive ALL 8 dp ranks (16 distinct slots), not a - # stride-2 subset of 4. - routed = _route_n(route, _instances(2, 8), 2 * 8) - per_instance: dict[int, set] = {0: set(), 1: set()} - for inst, dp_rank in routed: - per_instance[inst].add(dp_rank) - assert per_instance[0] == set(range(8)) - assert per_instance[1] == set(range(8)) - - -def test_consecutive_requests_alternate_instances(route): - # Interleaved order spreads consecutive requests across instances rather - # than filling one instance's ranks before moving on. - insts = [inst for inst, _ in _route_n(route, _instances(3, 8), 3)] - assert insts == [0, 1, 2] - - -@pytest.mark.skipif(sys.platform != "linux", reason="GIL probe uses libc.usleep") -def test_discovery_heartbeat_progresses_while_worker_holds_gil(): - from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_heartbeat import ( - MoRIIOHeartbeat, - ) - - payload = { - "type": "P", - "http_address": "127.0.0.1:8000", - "zmq_address": "host:127.0.0.1,handshake:6301,notify:61005", - "dp_size": 1, - "tp_size": 8, - "transfer_mode": "READ", - } - with zmq.Context() as context, context.socket(zmq.ROUTER) as socket: - port = socket.bind_to_random_port("tcp://127.0.0.1") - heartbeat = MoRIIOHeartbeat(f"tcp://127.0.0.1:{port}", payload, 0.02) - try: - assert socket.poll(5000), "Discovery did not register" - assert msgpack.unpackb(socket.recv_multipart()[1]) == payload - # PyDLL retains the GIL throughout the native call. - ctypes.PyDLL(None).usleep(300_000) - received = 0 - while socket.poll(0): - assert msgpack.unpackb(socket.recv_multipart()[1]) == payload - received += 1 - assert received >= 3, "Discovery stalled while the worker held the GIL" - finally: - heartbeat.shutdown() - heartbeat.shutdown() - while socket.poll(0): - socket.recv_multipart() - assert not socket.poll(100), "Heartbeats continued after shutdown" - - -@pytest.mark.skipif(sys.platform != "linux", reason="Checks child exit via procfs") -def test_discovery_heartbeat_exits_when_worker_dies(): - from vllm.distributed.kv_transfer.kv_connector.v1.moriio import moriio_heartbeat - - owner_code = """ -import runpy -import sys -Heartbeat = runpy.run_path(sys.argv[1])["MoRIIOHeartbeat"] -heartbeat = Heartbeat(sys.argv[2], {"type": "D"}, 0.02) -print(heartbeat._process.pid, flush=True) -sys.stdin.read() -""" - with zmq.Context() as context, context.socket(zmq.ROUTER) as socket: - port = socket.bind_to_random_port("tcp://127.0.0.1") - owner = subprocess.Popen( - [ - sys.executable, - "-c", - owner_code, - moriio_heartbeat.__file__, - f"tcp://127.0.0.1:{port}", - ], - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - text=True, - ) - child_pid = None - child_exited = False - try: - assert socket.poll(5000), "Discovery did not register" - assert owner.stdout is not None - child_pid = int(owner.stdout.readline()) - owner.kill() - owner.wait(timeout=5) - stat = Path(f"/proc/{child_pid}/stat") - deadline = time.monotonic() + 5 - while stat.exists(): - # A terminated child may await reaping by the container's init. - try: - if stat.read_text().split(") ")[1].startswith("Z "): - break - except FileNotFoundError: - break - assert time.monotonic() < deadline, "Orphan heartbeat survived worker" - time.sleep(0.01) - child_exited = True - finally: - if owner.poll() is None: - owner.kill() - owner.communicate(timeout=5) - if child_pid is not None and not child_exited: - with contextlib.suppress(ProcessLookupError): - os.kill(child_pid, 9) - - -def test_discovery_heartbeat_recovers_after_router_outage(capfd): - from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_heartbeat import ( - MoRIIOHeartbeat, - ) - - payload = {"type": "P", "http_address": "127.0.0.1:8000"} - with zmq.Context() as context: - with context.socket(zmq.ROUTER) as socket: - port = socket.bind_to_random_port("tcp://127.0.0.1") - endpoint = f"tcp://127.0.0.1:{port}" - heartbeat = MoRIIOHeartbeat(endpoint, payload, 0.001, max_retries=1) - try: - assert socket.poll(5000), "Discovery did not register" - assert msgpack.unpackb(socket.recv_multipart()[1]) == payload - except BaseException: - heartbeat.shutdown() - raise - try: - # Fill the real DEALER queue while discovery is unavailable. - deadline = time.monotonic() + 10 - errors = "" - while "heartbeat send failed" not in errors: - errors += capfd.readouterr().err - assert time.monotonic() < deadline, "No send timeout during outage" - time.sleep(0.01) - with context.socket(zmq.ROUTER) as socket: - socket.bind(endpoint) - assert socket.poll(5000), "Discovery did not recover after outage" - assert msgpack.unpackb(socket.recv_multipart()[1]) == payload - assert heartbeat._process.poll() is None - finally: - heartbeat.shutdown() diff --git a/tests/v1/kv_connector/unit/test_moriio_routing.py b/tests/v1/kv_connector/unit/test_moriio_routing.py deleted file mode 100644 index b0ac6df3cb..0000000000 --- a/tests/v1/kv_connector/unit/test_moriio_routing.py +++ /dev/null @@ -1,98 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Unit tests for MoRIIO router-authoritative DP-rank routing. - -These lock in the redesign that addresses the PR #45043 review feedback -(njhill): the connector must NOT self-derive a DP rank, and the prefill leg -must *return* the rank it actually ran on via ``request_finished`` so the -decode->prefill notify is routed by pure propagation (READ / serial WRITE), -not by an independent hash on each side. - -Following test_moriio_unmap.py, we bind the unbound method to a lightweight -stand-in rather than constructing a full scheduler. -""" - -from types import SimpleNamespace - -from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_connector import ( - MoRIIOConnectorScheduler, -) -from vllm.v1.request import RequestStatus - -_request_finished = MoRIIOConnectorScheduler.request_finished -_clip_blocks = MoRIIOConnectorScheduler.get_exchange_clipped_blocks - - -def _producer(global_dp_rank: int, dp_size: int, dp_size_local: int): - """Minimal prefill-side (producer) scheduler stand-in for request_finished.""" - s = SimpleNamespace( - is_producer=True, - _is_hma_required=False, - _has_mamba=False, - blocks_per_sw=[0], - _global_dp_rank=global_dp_rank, - engine_id="engine-abc", - host_ip="10.0.0.1", - handshake_port=6300, - side_notify_port=7300, - vllm_config=SimpleNamespace( - parallel_config=SimpleNamespace( - data_parallel_size=dp_size, - data_parallel_size_local=dp_size_local, - tensor_parallel_size=1, - ) - ), - ) - s.get_exchange_clipped_blocks = _clip_blocks.__get__(s) - return s - - -def _req( - request_id: str, params: dict | None, status=RequestStatus.FINISHED_LENGTH_CAPPED -): - return SimpleNamespace( - request_id=request_id, - kv_transfer_params=params, - status=status, - ) - - -def test_request_finished_returns_prefill_rank_for_propagation(): - # 2P2D: global rank 11 lives on pod 1 (ranks 8..15), local size 8. - s = _producer(global_dp_rank=11, dp_size=16, dp_size_local=8) - req = _req("r1", {"do_remote_decode": True, "transfer_id": "t1"}) - - delay_free, params = _request_finished(s, req, block_ids=[]) - - # The prefill leg returns the rank IT actually ran on, marked authoritative, - # so the router forwards it to decode and the notify propagates (no hashing). - assert params is not None - assert params["remote_dp_rank"] == 11 - assert params["remote_dp_rank_override"] is True - assert params["remote_dp_size"] == 16 - assert params["remote_dp_size_local"] == 8 - assert params["transfer_id"] == "t1" - # Empty block_ids => nothing deferred, blocks freed immediately. - assert delay_free is False - - -def test_request_finished_rank_tracks_the_actual_rank(): - # A different rank must be echoed verbatim -- the value is the producer's - # own rank, never a recomputation of the request id. - s = _producer(global_dp_rank=4, dp_size=8, dp_size_local=8) - req = _req("r2", {"do_remote_decode": True, "transfer_id": "t2"}) - - _, params = _request_finished(s, req, block_ids=[]) - - assert params["remote_dp_rank"] == 4 - assert params["remote_dp_size_local"] == 8 - - -def test_request_finished_no_kv_params_is_noop(): - s = _producer(global_dp_rank=0, dp_size=16, dp_size_local=8) - req = _req("r3", None) - - delay_free, params = _request_finished(s, req, block_ids=[]) - - assert delay_free is False - assert params is None diff --git a/tests/v1/kv_connector/unit/test_moriio_routing_fairness.py b/tests/v1/kv_connector/unit/test_moriio_routing_fairness.py deleted file mode 100644 index e38ec3f7e9..0000000000 --- a/tests/v1/kv_connector/unit/test_moriio_routing_fairness.py +++ /dev/null @@ -1,210 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Hardware-fair KV-read routing across MoRIIO heterogeneous P/D configs. - -Dependency-light (no GPU / ROCm / mori): builds bare ``MoRIIOConnectorWorker`` -instances via ``object.__new__`` and drives the REAL read-source routing -decision (``_resolve_read_source`` / ``_next_flex_tp_rank``) directly -- no -routing logic is re-implemented here, so a future change to those functions is -what these tests exercise. - -Scope -- the CONNECTOR (decode-side) read routing. The connector never selects -the prefill instance; the proxy does and hands the connector a ``remote_host`` + -``remote_dp_rank`` per request. The connector's only fairness lever is WHICH -prefill (dp, tp) rank each read targets, and the RFC's deployments collapse to -three real connector behaviours: - - symmetric TP (TP prefill + TP decode) -> decode tp_k reads prefill tp_k - flexible (TP prefill + DP decode) -> round-robin over prefill tp0..N-1 - owner DP (DP prefill, any decode) -> read the owner rank, tp0 - -These assume the proxy delivers a FAIR request stream (each prefill instance + -owner dp-rank an equal share) and verify the connector never re-introduces a -bottleneck. That proxy contract is checked separately against the real -``flat_interleaved_dp_route`` in the toy-proxy tests (PR #46115). - -A node runs one of two modes (8 GPUs each): - * TP8 -> (dp_size, tp_size) = (1, 8); MLA latent KV REPLICATED on all ranks. - * DP8EP -> (dp_size, tp_size) = (8, 1); KV PARTITIONED, one owner rank/request. -""" - -from collections import Counter -from dataclasses import dataclass -from unittest.mock import MagicMock - -import pytest - -from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_common import ( - ReqMeta, - get_port_offset, -) -from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_connector import ( - MoRIIOConnectorWorker, -) - -MODE_DIMS = {"TP8": (1, 8), "DP8EP": (8, 1)} # (dp_size, tp_size), 8 GPUs/node - - -@dataclass(frozen=True) -class PDConfig: - name: str - p_mode: str - d_mode: str - - @property - def p_dp(self) -> int: - return MODE_DIMS[self.p_mode][0] - - @property - def p_tp(self) -> int: - return MODE_DIMS[self.p_mode][1] - - @property - def d_dp(self) -> int: - return MODE_DIMS[self.d_mode][0] - - @property - def d_tp(self) -> int: - return MODE_DIMS[self.d_mode][1] - - @property - def n_prefill_gpus(self) -> int: - return self.p_dp * self.p_tp - - -CONFIGS = [ - PDConfig("1P_TP8:1D_TP8", "TP8", "TP8"), - PDConfig("2P_TP8:1D_DP8EP", "TP8", "DP8EP"), - PDConfig("2P_TP8:2D_TP8", "TP8", "TP8"), - PDConfig("2P_DP8EP:3D_DP8EP", "DP8EP", "DP8EP"), - PDConfig("2P_DP8EP:4D_TP8", "DP8EP", "TP8"), -] -CONFIG_IDS = [c.name for c in CONFIGS] - - -def make_decode_worker(*, world_size: int, tp_rank: int, dp_rank: int): - w = object.__new__(MoRIIOConnectorWorker) - w.world_size = world_size - w.tp_rank = tp_rank - w.dp_rank = dp_rank - w.use_mla = True - return w - - -def make_meta(*, p_tp: int, p_dp: int, remote_dp_rank: int, host: str = "phost0"): - return ReqMeta( - transfer_id="t", - local_block_ids=[1], - remote_block_ids=[2], - remote_host=host, - remote_port=1234, - remote_handshake_port=6301, - remote_notify_port=61005, - remote_engine_id=f"{host}:6301", - tp_size=p_tp, - remote_dp_size=p_dp, - remote_dp_rank=remote_dp_rank, - ) - - -def build_decode_workers(cfg: PDConfig) -> list: - """Decode workers that issue reads for one prefill instance. A TP decode - instance reads from every tp rank; a DP decode instance from each dp-rank - worker. Reused across requests so per-worker round-robin state advances.""" - if cfg.d_tp > 1: - return [ - make_decode_worker(world_size=cfg.d_tp, tp_rank=r, dp_rank=0) - for r in range(cfg.d_tp) - ] - return [ - make_decode_worker(world_size=1, tp_rank=0, dp_rank=d) for d in range(cfg.d_dp) - ] - - -def prefill_target_multiset(cfg: PDConfig, rounds: int) -> Counter: - """Drive the REAL _resolve_read_source over a fair input stream; tally which - prefill GPU each read targets. The only modelled assumption is the proxy - contract -- each owner dp-rank delivered equally (``for owner_dp in - range(p_dp)``); no proxy algorithm is reproduced.""" - workers = build_decode_workers(cfg) - hits: Counter = Counter() - for _ in range(rounds): - for owner_dp in range(cfg.p_dp): - for worker in workers: - meta = make_meta(p_tp=cfg.p_tp, p_dp=cfg.p_dp, remote_dp_rank=owner_dp) - chosen_tp, _flexible = worker._resolve_read_source(meta) - target = get_port_offset(owner_dp, chosen_tp, cfg.p_tp) - assert 0 <= target < cfg.n_prefill_gpus - hits[target] += 1 - return hits - - -@pytest.mark.parametrize("cfg", CONFIGS, ids=CONFIG_IDS) -def test_kv_read_load_is_hardware_fair(cfg: PDConfig) -> None: - # rounds a multiple of p_tp so the round-robin closes on an exact cycle. - hits = prefill_target_multiset(cfg, rounds=16) - gpus = set(range(cfg.n_prefill_gpus)) - assert set(hits) == gpus, f"unused prefill GPUs: {sorted(gpus - set(hits))}" - assert max(hits.values()) == min(hits.values()), dict(sorted(hits.items())) - - -@pytest.mark.parametrize("cfg", CONFIGS, ids=CONFIG_IDS) -def test_flexible_gate_fires_only_for_tp_prefill_dp_decode(cfg: PDConfig) -> None: - flags = set() - for worker in build_decode_workers(cfg): - meta = make_meta(p_tp=cfg.p_tp, p_dp=cfg.p_dp, remote_dp_rank=0) - _chosen, flexible = worker._resolve_read_source(meta) - flags.add(flexible) - is_mirror = cfg.d_tp == 1 and cfg.p_dp == 1 and cfg.p_tp > 1 - assert flags == {is_mirror} - - -def test_symmetric_tp_is_a_bijection() -> None: - targets = [] - for tp_rank in range(8): - worker = make_decode_worker(world_size=8, tp_rank=tp_rank, dp_rank=0) - chosen_tp, flexible = worker._resolve_read_source( - make_meta(p_tp=8, p_dp=1, remote_dp_rank=0) - ) - assert not flexible - targets.append(chosen_tp) - assert sorted(targets) == list(range(8)) - - -def test_owner_dp_read_is_faithful_and_covers_every_rank() -> None: - worker = make_decode_worker(world_size=8, tp_rank=3, dp_rank=0) - targets = [] - for owner_dp in range(8): - chosen_tp, flexible = worker._resolve_read_source( - make_meta(p_tp=1, p_dp=8, remote_dp_rank=owner_dp) - ) - assert not flexible - assert chosen_tp == 0 # p_tp == 1, only tp0 exists - targets.append(get_port_offset(owner_dp, chosen_tp, 1)) - assert sorted(targets) == list(range(8)) - - -def test_flexible_round_robin_is_deterministic_uniform_and_staggered() -> None: - w0 = make_decode_worker(world_size=1, tp_rank=0, dp_rank=0) - seq = [w0._next_flex_tp_rank(8) for _ in range(64)] - assert seq[:8] == list(range(8)) - assert Counter(seq) == Counter({t: 8 for t in range(8)}) - - first_pick = [ - make_decode_worker(world_size=1, tp_rank=0, dp_rank=d)._next_flex_tp_rank(8) - for d in range(8) - ] - assert sorted(first_pick) == list(range(8)) - - -def test_read_blocks_for_req_threads_chosen_tp() -> None: - # The resolved (chosen_tp, flexible) must reach _read_blocks, which keys the - # session AND the notify port off that single value -- so a read and its - # completion notify address the same prefill rank. - worker = make_decode_worker(world_size=1, tp_rank=0, dp_rank=3) - worker._read_blocks = MagicMock() - worker._read_blocks_for_req("r", make_meta(p_tp=8, p_dp=1, remote_dp_rank=0)) - kw = worker._read_blocks.call_args.kwargs - assert kw["flexible"] is True - assert kw["chosen_tp"] == 3 # first flexible pick = dp_rank seed - assert get_port_offset(0, kw["chosen_tp"], 8) == 3 diff --git a/tests/v1/kv_connector/unit/test_moriio_tp_ack.py b/tests/v1/kv_connector/unit/test_moriio_tp_ack.py deleted file mode 100644 index 34d193e272..0000000000 --- a/tests/v1/kv_connector/unit/test_moriio_tp_ack.py +++ /dev/null @@ -1,494 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -import threading -from types import SimpleNamespace - -import pytest - -from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_common import ( - MoRIIOMode, - MoRIIOTransferAck, - TransferError, -) -from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_connector import ( - MoRIIOConnector, - MoRIIOConnectorWorker, - get_moriio_expected_ack_count, - get_moriio_remote_tp_rank, - resolve_moriio_transfer_ack, - validate_moriio_heterogeneous_tp_kv_heads, -) -from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_engine import ( - MoRIIOWrapper, -) - - -def test_remote_tp_rank_same_tp_maps_to_self(): - assert [get_moriio_remote_tp_rank(rank, 4, 4) for rank in range(4)] == [ - 0, - 1, - 2, - 3, - ] - - -def test_remote_tp_rank_p4_d8_floor_maps_decode_to_prefill(): - assert [get_moriio_remote_tp_rank(rank, 8, 4) for rank in range(8)] == [ - 0, - 0, - 1, - 1, - 2, - 2, - 3, - 3, - ] - - -def test_remote_tp_rank_p8_d4_maps_to_first_prefill_rank_per_pair(): - assert [get_moriio_remote_tp_rank(rank, 4, 8) for rank in range(4)] == [ - 0, - 2, - 4, - 6, - ] - - -@pytest.mark.parametrize( - ("local_tp_rank", "local_tp_size", "remote_tp_size"), - [ - (0, 6, 4), - (0, 4, 6), - ], -) -def test_remote_tp_rank_invalid_non_multiple_tp_raises( - local_tp_rank: int, local_tp_size: int, remote_tp_size: int -): - with pytest.raises(ValueError, match="multiple"): - get_moriio_remote_tp_rank(local_tp_rank, local_tp_size, remote_tp_size) - - -@pytest.mark.parametrize( - ("local_tp_size", "remote_tp_size", "total_num_kv_heads"), - [ - (4, 4, 8), - (8, 4, 4), - (4, 8, 4), - ], -) -def test_heterogeneous_tp_head_guard_allows_supported_layouts( - local_tp_size: int, remote_tp_size: int, total_num_kv_heads: int -): - validate_moriio_heterogeneous_tp_kv_heads( - local_tp_size, - remote_tp_size, - total_num_kv_heads, - is_mla=False, - ) - - -def test_heterogeneous_tp_head_guard_allows_mla_layouts(): - validate_moriio_heterogeneous_tp_kv_heads( - local_tp_size=2, - remote_tp_size=4, - total_num_kv_heads=4, - is_mla=True, - ) - - -@pytest.mark.parametrize( - ("local_tp_size", "remote_tp_size", "total_num_kv_heads"), - [ - (4, 2, 4), - (2, 4, 4), - ], -) -def test_heterogeneous_tp_head_guard_rejects_split_kv_heads( - local_tp_size: int, remote_tp_size: int, total_num_kv_heads: int -): - with pytest.raises(NotImplementedError, match="replicated KV heads"): - validate_moriio_heterogeneous_tp_kv_heads( - local_tp_size, - remote_tp_size, - total_num_kv_heads, - is_mla=False, - ) - - -def test_expected_ack_count_for_homogeneous_or_smaller_consumer_tp_is_one(): - assert get_moriio_expected_ack_count(4, 4) == 1 - assert get_moriio_expected_ack_count(8, 4) == 1 - - -def test_expected_ack_count_for_decode_fan_in(): - assert get_moriio_expected_ack_count(4, 8) == 2 - - -def test_expected_ack_count_rejects_non_multiple_fan_in(): - with pytest.raises(ValueError, match="multiple"): - get_moriio_expected_ack_count(4, 6) - - -def test_plain_string_ack_is_backward_compatible_single_ack(): - notification_counts: dict[str, int] = {} - completed_transfer_ids: set[str] = set() - - assert ( - resolve_moriio_transfer_ack( - "tx-plain", - producer_tp_size=4, - live_transfer_ids={"tx-plain"}, - notification_counts=notification_counts, - completed_transfer_ids=completed_transfer_ids, - ) - == "tx-plain" - ) - assert notification_counts == {} - assert completed_transfer_ids == {"tx-plain"} - - -def test_structured_release_ack_waits_for_all_expected_acks(): - ack = MoRIIOTransferAck("tx-fanin", consumer_tp_size=8) - notification_counts: dict[str, int] = {} - completed_transfer_ids: set[str] = set() - - assert ( - resolve_moriio_transfer_ack( - ack, - producer_tp_size=4, - live_transfer_ids={"tx-fanin"}, - notification_counts=notification_counts, - completed_transfer_ids=completed_transfer_ids, - ) - is None - ) - assert notification_counts == {"tx-fanin": 1} - assert completed_transfer_ids == set() - - assert ( - resolve_moriio_transfer_ack( - ack, - producer_tp_size=4, - live_transfer_ids={"tx-fanin"}, - notification_counts=notification_counts, - completed_transfer_ids=completed_transfer_ids, - ) - == "tx-fanin" - ) - assert notification_counts == {} - assert completed_transfer_ids == {"tx-fanin"} - - -def test_duplicate_ack_after_completion_does_not_resolve_twice(): - ack = MoRIIOTransferAck("tx-dup", consumer_tp_size=8) - notification_counts: dict[str, int] = {} - completed_transfer_ids: set[str] = set() - - assert ( - resolve_moriio_transfer_ack( - ack, - producer_tp_size=4, - live_transfer_ids={"tx-dup"}, - notification_counts=notification_counts, - completed_transfer_ids=completed_transfer_ids, - ) - is None - ) - assert ( - resolve_moriio_transfer_ack( - ack, - producer_tp_size=4, - live_transfer_ids={"tx-dup"}, - notification_counts=notification_counts, - completed_transfer_ids=completed_transfer_ids, - ) - == "tx-dup" - ) - assert ( - resolve_moriio_transfer_ack( - ack, - producer_tp_size=4, - live_transfer_ids={"tx-dup"}, - notification_counts=notification_counts, - completed_transfer_ids=completed_transfer_ids, - ) - is None - ) - assert notification_counts == {} - assert completed_transfer_ids == {"tx-dup"} - - -def test_ack_for_non_live_transfer_is_ignored(): - notification_counts: dict[str, int] = {} - completed_transfer_ids: set[str] = set() - - assert ( - resolve_moriio_transfer_ack( - MoRIIOTransferAck("tx-stale", consumer_tp_size=8), - producer_tp_size=4, - live_transfer_ids={"tx-live"}, - notification_counts=notification_counts, - completed_transfer_ids=completed_transfer_ids, - ) - is None - ) - assert notification_counts == {} - assert completed_transfer_ids == set() - - -def test_worker_get_finished_counts_structured_release_fan_in(): - class FakeWrapper: - def __init__(self): - self.batches = [ - [MoRIIOTransferAck("tx-fanin", consumer_tp_size=8)], - [MoRIIOTransferAck("tx-fanin", consumer_tp_size=8)], - ] - - def pop_finished_req_ids(self): - return self.batches.pop(0) - - def shutdown(self): - pass - - worker = MoRIIOConnectorWorker.__new__(MoRIIOConnectorWorker) - worker.is_producer = True - worker.mode = MoRIIOMode.READ - worker.world_size = 4 - worker.moriio_wrapper = FakeWrapper() - worker.transfer_id_to_request_id = {"tx-fanin": "req-fanin"} - worker._consumer_notification_counts = {} - worker._completed_consumer_notifications = set() - worker._pending_unmapped_acks = [] - - assert worker.get_finished() == (set(), set()) - assert worker._consumer_notification_counts == {"tx-fanin": 1} - - assert worker.get_finished() == ({"req-fanin"}, set()) - assert worker._consumer_notification_counts == {} - assert worker._completed_consumer_notifications == {"tx-fanin"} - - -@pytest.mark.parametrize("exit_kind", ["killed", "eof"]) -def test_worker_reports_unexpected_heartbeat_exit(exit_kind): - from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_heartbeat import ( - MoRIIOHeartbeat, - ) - - worker = MoRIIOConnectorWorker.__new__(MoRIIOConnectorWorker) - worker.is_producer = True - worker.mode = MoRIIOMode.READ - worker.moriio_wrapper = SimpleNamespace( - pop_finished_req_ids=lambda: [], shutdown=lambda: None - ) - worker._pending_unmapped_acks = [] - worker._heartbeat = MoRIIOHeartbeat("tcp://127.0.0.1:1", {"type": "P"}, 0.02) - process = worker._heartbeat._process - try: - assert worker.get_finished() == (set(), set()) - if exit_kind == "killed": - process.kill() - else: - process.stdin.close() - returncode = process.wait(timeout=5) - with pytest.raises(RuntimeError, match=f"heartbeat.*{returncode}"): - worker.get_finished() - finally: - worker.shutdown() - - -def test_read_completion_sends_structured_release_with_consumer_tp_size(): - class DoneStatus: - def Succeeded(self): - return True - - def Failed(self): - return False - - class FakeWrapper: - def __init__(self): - self.lock = threading.Lock() - self.sent = [] - - # Exercise the real batch verdict rather than restating it here. - poll_transfer_batch = MoRIIOWrapper.poll_transfer_batch - - def send_notify( - self, - transfer_id, - host, - port, - message_type=None, - message_fields=None, - ): - self.sent.append((transfer_id, host, port, message_type, message_fields)) - - def shutdown(self): - pass - - worker = MoRIIOConnectorWorker.__new__(MoRIIOConnectorWorker) - worker.world_size = 8 - worker.moriio_config = SimpleNamespace(recv_abort_timeout=600.0) - worker.moriio_wrapper = FakeWrapper() - # A layer maps to the list of reads posted for it (a KDA layer posts two). - worker._recving_transfers = {"req": {"layer0": [DoneStatus(), DoneStatus()]}} - worker._recving_transfers_callback_addr = { - "req": ("127.0.0.1", "7000", "tx-release") - } - # Transfer-timeout reaping state consulted by _pop_done_transfers. - worker._recving_transfers_start = {} - # Load-error bookkeeping cleared alongside a completed transfer. - worker._recving_local_blocks = {} - worker._invalid_block_ids = set() - - assert worker._pop_done_transfers() == {"tx-release"} - assert worker.moriio_wrapper.sent == [ - ( - "tx-release", - "127.0.0.1", - "7000", - "release", - {"consumer_tp_size": 8}, - ) - ] - assert worker._recving_transfers == {} - assert worker._recving_transfers_callback_addr == {} - - -def test_aborted_read_without_local_blocks_only_releases_remote_blocks(): - class FakeWrapper: - def __init__(self): - self.sent = [] - - def send_notify( - self, - transfer_id, - host, - port, - message_type=None, - message_fields=None, - ): - self.sent.append((transfer_id, host, port, message_type, message_fields)) - - def shutdown(self): - pass - - worker = MoRIIOConnectorWorker.__new__(MoRIIOConnectorWorker) - worker.mode = MoRIIOMode.READ - worker.world_size = 8 - worker.moriio_wrapper = FakeWrapper() - - worker._read_blocks( - local_block_ids=[], - remote_block_ids=[[10, 11], [90]], - dst_engine_id="prefill", - request_id="req-aborted", - transfer_id="tx-aborted", - remote_host="127.0.0.1", - remote_notify_port=7000, - remote_tp_size=8, - remote_dp_rank=0, - chosen_tp=2, - ) - - assert worker.moriio_wrapper.sent == [ - ( - "tx-aborted", - "127.0.0.1", - "7002", - "release", - {"consumer_tp_size": 8}, - ) - ] - - -def test_read_completion_waits_for_every_posted_read(): - class Status: - def __init__(self, done: bool): - self.done = done - - def Succeeded(self): - return self.done - - def Failed(self): - return False - - class FakeWrapper: - def __init__(self): - self.lock = threading.Lock() - self.sent = [] - - poll_transfer_batch = MoRIIOWrapper.poll_transfer_batch - - def send_notify(self, *args, **kwargs): - self.sent.append((args, kwargs)) - - def shutdown(self): - pass - - worker = MoRIIOConnectorWorker.__new__(MoRIIOConnectorWorker) - worker.world_size = 8 - worker.moriio_config = SimpleNamespace(recv_abort_timeout=600.0) - worker.moriio_wrapper = FakeWrapper() - worker._recving_transfers = { - "req": {"kda_layer": [Status(done=True), Status(done=False)]} - } - worker._recving_transfers_callback_addr = { - "req": ("127.0.0.1", "7000", "tx-pending") - } - worker._recving_transfers_start = {"req": float("inf")} - worker._recving_local_blocks = {"req": [1, 2]} - worker._invalid_block_ids = set() - - assert worker._pop_done_transfers() == set() - assert "req" in worker._recving_transfers - assert worker.moriio_wrapper.sent == [] - - -@pytest.mark.parametrize( - ("has_mamba", "expected_invalid"), - [(False, {1, 2}), (True, set())], -) -def test_failed_read_reports_blocks_only_without_hma(has_mamba, expected_invalid): - worker = MoRIIOConnectorWorker.__new__(MoRIIOConnectorWorker) - worker._has_mamba = has_mamba - worker._recving_local_blocks = {"req": [1, 2]} - worker._invalid_block_ids = set() - - worker._record_failed_recv("req") - - assert worker.get_block_ids_with_load_errors() == expected_invalid - - -def test_hybrid_step_barrier_fails_closed(monkeypatch): - class FailingWrapper: - def waiting_for_transfer_complete(self, _statuses): - raise TransferError("failed") - - def shutdown(self): - pass - - worker = MoRIIOConnectorWorker.__new__(MoRIIOConnectorWorker) - worker._has_mamba = True - worker._reads_issued_this_step = [object()] - worker._mamba_reads_this_step = [object()] - worker.moriio_wrapper = FailingWrapper() - monkeypatch.setattr( - "vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_connector.get_forward_context", - lambda: SimpleNamespace(cudagraph_runtime_mode=None), - ) - - with pytest.raises(TransferError, match="failed"): - worker._await_reads_issued_this_step() - - -def test_requested_cudagraph_mode_is_never_overridden(): - # The configured cudagraph mode is always honored: the barrier fires when - # the operator sets cudagraph_mode=PIECEWISE, and READ mode with full - # graphs only warns instead of silently forcing PIECEWISE. - assert ( - MoRIIOConnector.requires_piecewise_for_cudagraph({"read_mode": True}) is False - ) - assert ( - MoRIIOConnector.requires_piecewise_for_cudagraph({"read_mode": False}) is False - ) diff --git a/tests/v1/kv_connector/unit/test_moriio_unmap.py b/tests/v1/kv_connector/unit/test_moriio_unmap.py deleted file mode 100644 index b4fd4b8813..0000000000 --- a/tests/v1/kv_connector/unit/test_moriio_unmap.py +++ /dev/null @@ -1,68 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Unit tests for MoRIIOConnectorScheduler.{map,unmap}_request_id. - -vLLM's input_processor mutates a request_id after the connector has stored it -(e.g. ``cmpl-abc-0`` -> ``cmpl-abc-0-956053a4``), so an exact-match unmap can -miss. These tests cover the stable-``transfer_id`` fallback that handles that -case (and the clean exact-match and miss paths). - -The map/unmap methods touch only the two id<->id dicts, so we bind them to a -lightweight stand-in rather than constructing a full scheduler. -""" - -from types import SimpleNamespace - -from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_connector import ( - MoRIIOConnectorScheduler, -) - -_map = MoRIIOConnectorScheduler.map_request_id -_unmap = MoRIIOConnectorScheduler.unmap_request_id - - -def _sched(): - return SimpleNamespace( - transfer_id_to_request_id={}, - request_id_to_transfer_id={}, - ) - - -def test_exact_match_unmap_clears_both_tables(): - s = _sched() - _map(s, "rid-1", "tid-1") - _unmap(s, "rid-1") - assert s.request_id_to_transfer_id == {} - assert s.transfer_id_to_request_id == {} - - -def test_unmap_via_transfer_id_after_request_id_mutation(): - s = _sched() - # Connector stores the ORIGINAL request_id at map time. - _map(s, "cmpl-abc-0", "tid-42") - # input_processor later mutates the rid; exact match would miss, so the - # stable transfer_id is used to find and clear the original entry. - _unmap(s, "cmpl-abc-0-956053a4", transfer_id="tid-42") - assert s.request_id_to_transfer_id == {} - assert s.transfer_id_to_request_id == {} - - -def test_unmap_miss_is_noop_and_does_not_raise(): - s = _sched() - _map(s, "rid-1", "tid-1") - # Unknown rid and unknown/None transfer_id: must not raise, must not touch - # the existing (unrelated) mapping. - _unmap(s, "rid-unknown", transfer_id=None) - _unmap(s, "rid-unknown", transfer_id="tid-unknown") - assert s.request_id_to_transfer_id == {"rid-1": "tid-1"} - assert s.transfer_id_to_request_id == {"tid-1": "rid-1"} - - -def test_exact_match_preferred_over_transfer_id_fallback(): - s = _sched() - _map(s, "rid-1", "tid-1") - # Exact rid is present, so it is removed directly even if a transfer_id is - # also supplied; no stale entries left behind. - _unmap(s, "rid-1", transfer_id="tid-1") - assert s.request_id_to_transfer_id == {} - assert s.transfer_id_to_request_id == {} diff --git a/tests/v1/kv_connector/unit/test_moriio_write_done_routing.py b/tests/v1/kv_connector/unit/test_moriio_write_done_routing.py deleted file mode 100644 index ee04c2cabe..0000000000 --- a/tests/v1/kv_connector/unit/test_moriio_write_done_routing.py +++ /dev/null @@ -1,449 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Unit tests for mori-io write_done routing (#51681).""" - -import threading -import time -from collections import OrderedDict -from types import SimpleNamespace -from typing import Any -from unittest.mock import patch - -import pytest - -from vllm.distributed.kv_transfer.kv_connector.v1.moriio import moriio_engine -from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_common import ( - ROLE, - LayerTransferPlan, - MoRIIOError, - MoRIIOMode, - RemoteAllocInfo, - ReqMeta, - WriteTask, - get_port_offset, -) -from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_connector import ( - MoRIIOConnectorScheduler, - MoRIIOConnectorWorker, -) -from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_engine import ( - MoRIIOWrapper, -) - -from .utils import make_moriio_writer - -pytestmark = pytest.mark.cpu_test - - -def _fake_worker( - alloc: dict[str, Any], - *, - tp_rank: int = 0, - set_remote_dp_size_local: int | None = None, -) -> Any: - wrapper = SimpleNamespace( - done_remote_allocate_req_dict=alloc, - lock=threading.Lock(), - done_req_ids=[], - sent_notifies=[], - ) - wrapper.send_notify = lambda transfer_id, remote_ip, remote_port, message_type: ( - wrapper.sent_notifies.append( - (transfer_id, remote_ip, remote_port, message_type) - ) - ) - wrapper.waiting_for_transfer_complete = lambda statuses: None - wrapper._mark_transfer_terminal_locked = lambda transfer_id: None - wrapper._is_transfer_terminal_locked = lambda transfer_id: False - worker = SimpleNamespace(moriio_wrapper=wrapper, tp_rank=tp_rank) - if set_remote_dp_size_local is not None: - worker.remote_dp_size_local = set_remote_dp_size_local - return worker - - -def _req_meta( - *, - transfer_id: str = "tA", - remote_host: str = "10.0.0.1", - multi_pod_hosts: list[str] | None = None, - remote_dp_size: int = 16, - remote_dp_size_local: int = 8, -) -> ReqMeta: - return ReqMeta( - transfer_id=transfer_id, - local_block_ids=[[1]], - remote_block_ids=[[2]], - remote_host=remote_host, - remote_port=1, - remote_handshake_port=2, - remote_notify_port=20000, - remote_engine_id="remote-engine", - tp_size=1, - remote_dp_size=remote_dp_size, - multi_pod_hosts=list(multi_pod_hosts or []), - remote_dp_size_local=remote_dp_size_local, - ) - - -def _write_task( - transfer_id: str, - remote_ip: str, - *, - multi_pod_hosts: list[str] | None = None, - remote_dp_size_local: int = 0, - remote_dp_size: int = 1, - request_id: str = "req", -) -> WriteTask: - return WriteTask( - request_id=request_id, - transfer_id=transfer_id, - dst_engine_id="remote-engine", - local_block_ids=[[1, 3]], - remote_block_ids_hint=None, - layer_name="layer0", - event=None, - remote_notify_port=20000, - remote_ip=remote_ip, - multi_pod_hosts=multi_pod_hosts or [], - remote_dp_size_local=remote_dp_size_local, - remote_dp_size=remote_dp_size, - ) - - -def _real_wrapper() -> MoRIIOWrapper: - wrapper = MoRIIOWrapper.__new__(MoRIIOWrapper) - wrapper.lock = threading.Lock() - wrapper.done_req_ids = [] - wrapper.done_remote_allocate_req_dict = {} - wrapper.done_write_cache_req_ids = [] - wrapper.failed_write_cache_req_ids = [] - wrapper._terminal_transfer_ids = OrderedDict() - return wrapper - - -def test_execute_write_task_ip_is_per_task_under_overwrite(): - alloc = {"tA": RemoteAllocInfo(block_ids=None, decode_dp_rank=0)} - worker = _fake_worker(alloc) - worker.multi_pod_hosts = ["10.0.0.2"] - worker.remote_dp_size_local = 1 - - writer = make_moriio_writer(worker) - task_a = _write_task( - "tA", "10.0.0.1", multi_pod_hosts=["10.0.0.1"], remote_dp_size_local=1 - ) - writer._execute_write_task(task_a) - - assert alloc["tA"].completion_remote_ip == "10.0.0.1" - assert alloc["tA"].completion_remote_ip != "10.0.0.2" - - -def test_deferred_task_ip_is_per_task_under_overwrite(): - alloc = {"tA": RemoteAllocInfo(block_ids=None, decode_dp_rank=0)} - worker = _fake_worker(alloc) - worker.multi_pod_hosts = ["10.0.0.2"] - worker.remote_dp_size_local = 1 - - writer = make_moriio_writer(worker) - task_a = _write_task( - "tA", "10.0.0.1", multi_pod_hosts=["10.0.0.1"], remote_dp_size_local=1 - ) - writer._deferred_tasks.append(task_a) - writer._process_deferred_tasks() - - assert alloc["tA"].completion_remote_ip == "10.0.0.1" - assert writer._deferred_tasks == [] - - -def test_two_requests_resolve_distinct_hosts(): - alloc = { - "tA": RemoteAllocInfo(block_ids=None, decode_dp_rank=0), - "tB": RemoteAllocInfo(block_ids=None, decode_dp_rank=3), - } - worker = _fake_worker(alloc) - writer = make_moriio_writer(worker) - - task_a = _write_task( - "tA", "10.0.0.1", multi_pod_hosts=["10.0.0.1"], remote_dp_size_local=1 - ) - task_b = _write_task( - "tB", "10.0.0.2", multi_pod_hosts=["10.0.0.2"], remote_dp_size_local=1 - ) - worker.multi_pod_hosts = ["10.0.0.2"] - writer._execute_write_task(task_a) - writer._execute_write_task(task_b) - - assert alloc["tA"].completion_remote_ip == "10.0.0.1" - assert alloc["tB"].completion_remote_ip == "10.0.0.2" - - -def test_finalize_notify_port_uses_pod_local_rank(): - base = 20000 - info = RemoteAllocInfo(block_ids=None, decode_dp_rank=9) - alloc = {"tA": info} - worker = _fake_worker(alloc, tp_rank=0) - writer = make_moriio_writer(worker) - - task = _write_task( - "tA", - "10.0.0.1", - multi_pod_hosts=["10.0.0.1", "10.0.0.2"], - remote_dp_size_local=8, - ) - writer._execute_write_task(task) - assert info.completion_remote_ip == "10.0.0.2" - - info.writes_expected = 1 - info.writes_done = 1 - writer._finalize_if_complete("tA", info) - - assert worker.moriio_wrapper.sent_notifies - _tid, ip, port, mtype = worker.moriio_wrapper.sent_notifies[-1] - assert mtype == "write_done" - assert ip == "10.0.0.2" - assert port == base + get_port_offset(9 % 8, 0) - - -def test_finalize_port_ignores_stale_worker_dp_local(): - base = 20000 - info = RemoteAllocInfo(block_ids=None, decode_dp_rank=9) - alloc = {"tA": info} - worker = _fake_worker(alloc, tp_rank=0, set_remote_dp_size_local=1) - writer = make_moriio_writer(worker) - - task = _write_task( - "tA", - "10.0.0.1", - multi_pod_hosts=["10.0.0.1", "10.0.0.2"], - remote_dp_size_local=8, - ) - writer._execute_write_task(task) - assert info.completion_notify_port == base + get_port_offset(9 % 8, 0) - - info.writes_expected = 1 - info.writes_done = 1 - writer._finalize_if_complete("tA", info) - - _tid, _ip, port, mtype = worker.moriio_wrapper.sent_notifies[-1] - assert mtype == "write_done" - assert port == base + get_port_offset(9 % 8, 0) - assert port != base + get_port_offset(9 % 1, 0) - - -def test_finalize_single_pod_port_uses_global_rank(): - base = 20000 - info = RemoteAllocInfo(block_ids=None, decode_dp_rank=3) - alloc = {"tA": info} - worker = _fake_worker(alloc, tp_rank=2) - writer = make_moriio_writer(worker) - - task = _write_task("tA", "10.0.0.1", multi_pod_hosts=[], remote_dp_size_local=0) - writer._execute_write_task(task) - assert info.completion_remote_ip == "10.0.0.1" - assert info.completion_notify_port == base + get_port_offset(3, 2) - - info.writes_expected = 1 - info.writes_done = 1 - writer._finalize_if_complete("tA", info) - - _tid, _ip, port, _mtype = worker.moriio_wrapper.sent_notifies[-1] - assert port == base + get_port_offset(3, 2) - - -def test_write_blocks_for_req_forwards_routing_to_schedule_write(): - worker = MoRIIOConnectorWorker.__new__(MoRIIOConnectorWorker) - worker._has_mamba = False - captured: dict[str, Any] = {} - - def _capture_schedule_write_blocks(**kwargs: Any) -> None: - captured.update(kwargs) - - worker.schedule_write_blocks = _capture_schedule_write_blocks # type: ignore[method-assign] - - meta = _req_meta( - remote_host="10.0.0.1", - multi_pod_hosts=["10.0.0.1", "10.0.0.2"], - remote_dp_size_local=8, - ) - worker._write_blocks_for_req("req-a", meta, "layer0", kv_layer=None) - - assert captured["multi_pod_hosts"] == ["10.0.0.1", "10.0.0.2"] - assert captured["remote_dp_size_local"] == 8 - assert captured["remote_ip"] == "10.0.0.1" - assert captured["local_block_ids"] == [[1]] - assert captured["remote_block_ids"] == [[2]] - assert not hasattr(worker, "multi_pod_hosts") - - -def test_write_blocks_for_req_falls_back_when_multi_pod_hosts_empty(): - worker = MoRIIOConnectorWorker.__new__(MoRIIOConnectorWorker) - worker._has_mamba = False - captured: dict[str, Any] = {} - - def _capture_schedule_write_blocks(**kwargs: Any) -> None: - captured.update(kwargs) - - worker.schedule_write_blocks = _capture_schedule_write_blocks # type: ignore[method-assign] - - meta = _req_meta( - remote_host="10.0.0.9", - multi_pod_hosts=[], - remote_dp_size=4, - remote_dp_size_local=0, - ) - worker._write_blocks_for_req("req-b", meta, "layer0", kv_layer=None) - - assert captured["multi_pod_hosts"] == ["10.0.0.9"] - assert captured["remote_dp_size_local"] == 4 - assert captured["local_block_ids"] == [[1]] - assert captured["remote_block_ids"] == [[2]] - - -def test_deferred_write_timeout_fails_transfer_on_every_decode_rank(): - wrapper = _real_wrapper() - sent: list[tuple[str, int, str]] = [] - wrapper.send_notify = lambda tid, ip, port, message_type: sent.append( - (ip, port, message_type) - ) - writer = make_moriio_writer(SimpleNamespace(moriio_wrapper=wrapper, tp_rank=0)) - task = _write_task( - "tA", - "10.0.0.1", - multi_pod_hosts=["10.0.0.1", "10.0.0.2"], - remote_dp_size_local=2, - remote_dp_size=4, - ) - task.enqueue_time = time.perf_counter() - 2 * writer._defer_timeout - writer._deferred_tasks.append(task) - - writer._process_deferred_tasks() - - assert writer._deferred_tasks == [] - assert [ack.transfer_id for ack in wrapper.done_req_ids] == ["tA"] - # Global DP ranks 0-3, two per pod: the owning rank is unknown. - assert sent == [ - (host, 20000 + get_port_offset(local_rank, 0), "write_failed") - for host in ("10.0.0.1", "10.0.0.2") - for local_rank in (0, 1) - ] - - # A late decode allocation must not resurrect a write into freed blocks. - with patch.object(moriio_engine, "get_role", return_value=ROLE.PRODUCER): - wrapper._handle_remote_blocks_message( - {"transfer_id": "tA", "block_notify_list": [7], "decode_rank": 1} - ) - assert wrapper.done_remote_allocate_req_dict == {} - - -def test_write_scheduler_waits_for_worker_ack_past_defer_deadline(): - scheduler = SimpleNamespace( - is_producer=True, - mode=MoRIIOMode.WRITE, - _defer_timeout=60.0, - _deferred_send_deadlines={"req-1": (time.monotonic() - 1.0, "tA")}, - _pending_sent_acks={}, - unmap_request_id=lambda *args, **kwargs: None, - ) - output = SimpleNamespace(finished_sending=set()) - - MoRIIOConnectorScheduler.update_connector_output(scheduler, output) - assert output.finished_sending is None - assert "req-1" in scheduler._deferred_send_deadlines - - output.finished_sending = {"req-1"} - MoRIIOConnectorScheduler.update_connector_output(scheduler, output) - assert output.finished_sending == {"req-1"} - assert scheduler._deferred_send_deadlines == {} - - -def test_write_failed_is_reported_as_failed_recving_on_owning_rank(): - wrapper = _real_wrapper() - with patch.object(moriio_engine, "get_role", return_value=ROLE.CONSUMER): - for tid in ("tA", "tOtherRank"): - wrapper._handle_structured_message( - {"type": "write_failed", "transfer_id": tid} - ) - worker = SimpleNamespace( - mode=MoRIIOMode.WRITE, - is_producer=False, - moriio_wrapper=wrapper, - transfer_id_to_request_id={"tA": "req-1"}, - _unmatched_write_failures=OrderedDict(), - get_finished=lambda: (set(), set()), - ) - - results = MoRIIOConnectorWorker.get_transfer_results(worker) - - assert results.finished_recving == {"req-1"} - assert results.failed_recving == {"req-1"} - assert wrapper.pop_finished_write_req_ids() == set() - assert list(worker._unmatched_write_failures) == ["tOtherRank"] - - -def test_hybrid_write_rejects_unequal_tp_before_scheduling_any_layer(): - from unittest.mock import Mock - - worker = MoRIIOConnectorWorker.__new__(MoRIIOConnectorWorker) - worker._has_mamba = True - worker.world_size = 2 - worker.schedule_write_blocks = Mock() - - with pytest.raises(NotImplementedError, match="heterogeneous-TP"): - worker._write_blocks_for_req("req", _req_meta(), "attention", kv_layer=None) - - worker.schedule_write_blocks.assert_not_called() - - -@pytest.mark.parametrize( - "notify,expected", - [ - # Plain WRITE peers still send one flat attention list. - ([7, 8], [[7, 8]]), - # Hybrid WRITE sends one list per transfer group, kept as-is even - # when the first group is empty (full attention hit, state only). - ([[7], [9]], [[7], [9]]), - ([[], [9]], [[], [9]]), - ], -) -def test_remote_blocks_message_normalizes_to_grouped_block_ids(notify, expected): - wrapper = _real_wrapper() - with patch.object(moriio_engine, "get_role", return_value=ROLE.PRODUCER): - wrapper._handle_remote_blocks_message( - {"transfer_id": "tA", "block_notify_list": notify, "decode_rank": 2} - ) - info = wrapper.done_remote_allocate_req_dict["tA"] - assert info.block_ids == expected - assert info.decode_dp_rank == 2 - - -def test_empty_layer_plan_posts_no_rdma_write(): - from unittest.mock import Mock - - wrapper = SimpleNamespace(write_remote_data=Mock(), write_remote_data_single=Mock()) - writer = make_moriio_writer(SimpleNamespace(moriio_wrapper=wrapper)) - plan = LayerTransferPlan( - request_id="req", - transfer_id="tA", - layer_name="layer0", - sess_idx=0, - transfer_local_offsets=[], - transfer_remote_offsets=[], - transfer_sizes=[], - ) - - assert writer._do_layer_write(plan, ["session"]) == [] - wrapper.write_remote_data.assert_not_called() - wrapper.write_remote_data_single.assert_not_called() - - -@pytest.mark.parametrize("local_state", [[], [1, 2]]) -def test_hybrid_write_requires_exactly_one_running_state_slot(local_state): - worker = SimpleNamespace( - _mamba_blocks_for_layer=lambda layer_name, block_ids: block_ids[1], - ) - writer = make_moriio_writer(worker) - task = _write_task("tA", "10.0.0.1") - task.layer_name = "kda" - task.local_block_ids = [[], local_state] - - with pytest.raises(MoRIIOError, match="one running-state slot"): - writer._prepare_mamba_transfer_plans(task, RemoteAllocInfo(block_ids=[[], [9]])) diff --git a/tests/v1/kv_connector/unit/test_nixl_connector.py b/tests/v1/kv_connector/unit/test_nixl_connector.py deleted file mode 100644 index f90de6297a..0000000000 --- a/tests/v1/kv_connector/unit/test_nixl_connector.py +++ /dev/null @@ -1,4820 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -import contextlib -import inspect -import os -import queue -import tempfile -import textwrap -import threading -import time -import uuid -from collections import defaultdict -from concurrent.futures import Future -from types import SimpleNamespace -from typing import Any, cast -from unittest.mock import MagicMock, call, patch - -import msgspec -import numpy as np -import pytest -import ray -import torch - -from tests.v1.attention.utils import dense_kv_cache_views -from vllm import LLM -from vllm.config import KVTransferConfig, set_current_vllm_config -from vllm.distributed.kv_transfer.kv_connector.utils import ( - EngineTransferInfo, - KVOutputAggregator, - TransferTopology, - get_current_attn_backend, -) -from vllm.distributed.kv_transfer.kv_connector.v1 import nixl -from vllm.distributed.kv_transfer.kv_connector.v1.base import ( - KVConnectorRole, - KVConnectorTransferResults, -) -from vllm.distributed.kv_transfer.kv_connector.v1.metrics import KVConnectorStats -from vllm.distributed.kv_transfer.kv_connector.v1.multi_connector import ( - MultiKVConnectorStats, -) -from vllm.distributed.kv_transfer.kv_connector.v1.nixl import ( - NixlAgentMetadata, - NixlConnector, - NixlConnectorMetadata, - NixlConnectorScheduler, - NixlConnectorWorker, - NixlHandshakePayload, - NixlKVConnectorStats, -) -from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import ( - HeartbeatInfo, - RemoteMeta, - ReqMeta, - compute_nixl_compatibility_hash, -) -from vllm.distributed.kv_transfer.kv_transfer_state import ( - ensure_kv_transfer_shutdown, - has_kv_transfer_group, -) -from vllm.forward_context import ForwardContext -from vllm.outputs import RequestOutput -from vllm.platforms import current_platform -from vllm.platforms.interface import Platform -from vllm.sampling_params import SamplingParams -from vllm.v1.engine import EngineCoreRequest -from vllm.v1.engine.output_processor import OutputProcessor -from vllm.v1.kv_cache_interface import ( - AttentionSpec, - CircularBufferSpec, - FullAttentionSpec, - KVCacheConfig, - KVCacheGroupSpec, - KVCacheLayout, - KVQuantMode, - MLAAttentionSpec, - compute_layer_kv_cache_shape_bytes, -) -from vllm.v1.outputs import KVConnectorOutput, ModelRunnerOutput -from vllm.v1.request import RequestStatus - -from .utils import ( - create_model_runner_output, - create_request, - create_scheduler, - create_vllm_config, - expand_strided_descs, - make_kv_cache_config, - maybe_update_block_size, -) - - -@pytest.fixture(scope="module", autouse=True) -def clear_kv_transfer(): - """The test cases in this file use `VLLM_ENABLE_V1_MULTIPROCESSING=0`, - causing the global variable `_KV_CONNECTOR_AGENT` - to be assigned but never deleted. - - Since the current pytest process does not terminate and instead - continues running tests from other files, - this global variable remains in memory and interferes - with test cases in other modules. - - So we use this fixture to ensure that the global variable - `_KV_CONNECTOR_AGENT` is properly cleaned up after each test. - """ - yield - if has_kv_transfer_group(): - ensure_kv_transfer_shutdown() - - -def get_default_xfer_telemetry( - xferDurationS: float = 1, - postDurationS: float = 1, - totalBytes: int = 1, - descCount: int = 1, -) -> dict: - class AttributeDict(dict): - __slots__ = () - __getattr__ = dict.__getitem__ - __setattr__ = dict.__setitem__ - - # We can't instantiate nixlXferTelemetry because it's read only and - # ray env does not have NIXL, so we must fake it - return AttributeDict( - xferDuration=xferDurationS * 1e6, # in us - postDuration=postDurationS * 1e6, # in us - totalBytes=totalBytes, - descCount=descCount, - ) - - -class FakeNixlWrapper: - """Mock implementation of NixlWrapper for testing. - - We don't inherit from nixl._api.nixl_agent because nixl may not be - installed. - - Note: The complete source of this class is also used in the - `_make_fake_nixl_pkg` function to create a fake nixl package - for Ray workers. - """ - - AGENT_METADATA = b"fake_agent_metadata" - REMOTE_AGENT_NAME = "remote_agent" - - def __init__(self, agent_name: str, *args, **kwargs): - self._cycles_before_xfer_done = 0 - self._check_xfer_state_cycles: defaultdict[int, int] = defaultdict(lambda: 0) - self.invalid_remote_agents: set[str] = set() - - def get_reg_descs(self, caches_data, memory_type: str) -> list: - return [str(uuid.uuid4()) for _ in caches_data] - - def register_memory(self, descs, backends) -> None: - pass - - def deregister_memory(self, descs) -> None: - pass - - def get_xfer_descs(self, blocks_data, memory_type: str) -> list: - return [str(uuid.uuid4()) for _ in blocks_data] - - def prep_xfer_dlist( - self, agent_name: str, descs, mem_type: str | None = None - ) -> int: - return uuid.uuid4().int - - def get_agent_metadata(self) -> bytes: - return self.AGENT_METADATA - - def add_remote_agent(self, agent_metadata: bytes) -> str: - self.invalid_remote_agents.discard(self.REMOTE_AGENT_NAME) - return self.REMOTE_AGENT_NAME - - def get_new_notifs(self) -> dict[str, list[bytes]]: - # Used to collect done_sending, which we don't test yet. - return {} - - def check_xfer_state(self, handle: int) -> str: - if self._check_xfer_state_cycles[handle] >= self._cycles_before_xfer_done: - return "DONE" - self._check_xfer_state_cycles[handle] += 1 - return "PROC" - - def release_xfer_handle(self, handle: int) -> None: - pass - - def release_dlist_handle(self, handle: int) -> None: - pass - - def remove_remote_agent(self, agent: str) -> None: - pass - - def check_remote_metadata(self, agent: str) -> bool: - return agent not in self.invalid_remote_agents - - def send_notif(self, agent_name: str, notif_msg: bytes) -> None: - pass - - def make_prepped_xfer( - self, - xfer_type: str, - local_xfer_side_handle: int, - local_block_descs_ids: list[int], - remote_xfer_side_handle: int, - remote_block_descs_ids: list[int], - notif_msg: bytes | None = None, - ) -> int: - return uuid.uuid4().int - - def transfer(self, handle: int) -> str: - return "PROC" - - def get_xfer_telemetry(self, handle: int) -> dict: - return get_default_xfer_telemetry() - - -@contextlib.contextmanager -def _make_fake_nixl_pkg(): - """Context manager that creates a temporary package making - `from nixl._api import nixl_agent` resolve to our FakeNixlWrapper. - Also creates the ROCm NIXL packages. - - Automatically cleans up the temporary directory when done. - """ - with tempfile.TemporaryDirectory() as td: - for pkg_name in ["nixl", "nixl_rocm"]: - pkg_root = os.path.join(td, pkg_name, "_api") - os.makedirs(pkg_root, exist_ok=True) - - # Get the source code of FakeNixlWrapper class and dedent it - fake_nixl_source = inspect.getsource(FakeNixlWrapper) - fake_nixl_source = textwrap.dedent(fake_nixl_source) - - stub = f"""\ -# Copy of FakeNixlWrapper implementation for Ray workers -import uuid -from collections import defaultdict - -{fake_nixl_source} - -# Export as nixl_agent -nixl_agent = FakeNixlWrapper -""" - with open(os.path.join(pkg_root, "__init__.py"), "w") as f: - f.write(stub) - - # Mock nixlXferTelemetry class - pkg_root2 = os.path.join(td, pkg_name, "_bindings") - os.makedirs(pkg_root2, exist_ok=True) - with open(os.path.join(pkg_root2, "__init__.py"), "w") as f: - f.write("class nixlXferTelemetry: pass") - # touch parent package - open(os.path.join(td, pkg_name, "__init__.py"), "w").close() - - yield td - - -def test_basic_interface(): - """Unit test for basic NixlConnector interface functionality.""" - vllm_config = create_vllm_config() - scheduler = create_scheduler(vllm_config) - - # 2 Full Blocks and 1 Half Block. - BLOCK_SIZE = vllm_config.cache_config.block_size - NUM_EXTERNAL_FULL_BLOCKS = 2 - NUM_TOKENS = int(BLOCK_SIZE * (NUM_EXTERNAL_FULL_BLOCKS + 0.5)) - - request = create_request( - request_id=1, - block_size=BLOCK_SIZE, - num_tokens=NUM_TOKENS, - do_remote_prefill=True, - ) - request_id = request.request_id - - scheduler.add_request(request) - - # Remote Prefill, triggers NixlConnectorMetadata. - scheduler_output = scheduler.schedule() - kv_connector_metadata = scheduler_output.kv_connector_metadata - assert kv_connector_metadata is not None - assert isinstance(kv_connector_metadata, NixlConnectorMetadata) - - assert len(kv_connector_metadata.reqs_to_recv) == 1 - assert request_id in kv_connector_metadata.reqs_to_recv - req_meta = kv_connector_metadata.reqs_to_recv[request_id] - - for block_id, block in zip( - req_meta.local_block_ids[0], - scheduler.kv_cache_manager.coordinator.single_type_managers[0].req_to_blocks[ - request_id - ], - ): - assert block_id == block.block_id - - -def test_prompt_less_than_block_size(): - """Test that we can handle case where prompt is < block. - - In this case, the P worker will still send remote_block_ids of the - partial block. The D worker should schedule an async read - in this case. - """ - vllm_config = create_vllm_config() - scheduler = create_scheduler(vllm_config) - - # Half of a block. - BLOCK_SIZE = vllm_config.cache_config.block_size - NUM_TOKENS = int(BLOCK_SIZE * 0.5) - - # Request will have 1 partial remote block. - request = create_request( - request_id=1, - block_size=BLOCK_SIZE, - num_tokens=NUM_TOKENS, - do_remote_prefill=True, - num_remote_blocks=1, - ) - scheduler.add_request(request) - scheduler_output = scheduler.schedule() - - # This request will read async. - kv_connector_metadata = scheduler_output.kv_connector_metadata - assert kv_connector_metadata is not None - assert isinstance(kv_connector_metadata, NixlConnectorMetadata) - assert len(kv_connector_metadata.reqs_to_recv) == 1 - assert len(scheduler_output.scheduled_new_reqs) == 0 - - -def test_abort_immediately_remote_prefill_enqueues_empty_recv(): - """A remote-prefill request added with abort_immediately=True should - be added to the scheduler's waiting queue then immediately aborted, so the - NIXL connector's request_finished hook enqueues an empty recv to notify - the prefill instance to free its blocks.""" - from vllm.v1.request import RequestStatus - - scheduler = create_scheduler(create_vllm_config()) - - request = create_request(request_id=42, num_tokens=10, do_remote_prefill=True) - assert request.kv_transfer_params is not None - assert request.kv_transfer_params["do_remote_prefill"] is True - - # Mimic the EngineCore.add_request path for an abort-immediately req. - scheduler.add_request(request) - scheduler.finish_requests([request.request_id], RequestStatus.FINISHED_ABORTED) - - scheduler_output = scheduler.schedule() - meta = scheduler_output.kv_connector_metadata - assert isinstance(meta, NixlConnectorMetadata) - assert set(meta.reqs_to_recv) == {request.request_id} - req_meta = meta.reqs_to_recv[request.request_id] - assert req_meta.local_block_ids == [] - assert req_meta.remote.request_id == f"prefill-{42}" - # do_remote_prefill is consumed by request_finished to prevent re-issuing. - assert request.kv_transfer_params["do_remote_prefill"] is False - # The scheduler is not waiting on this recv -- the request is already gone - # from self.requests, so reporting it would trip `assert req_id in - # self.requests` in _update_from_kv_xfer_finished. - assert req_meta.awaiting_kvs is False - - -def test_prefill_exports_cached_tokens_in_kv_transfer_params(): - """The P worker reports its own prefix-cache hits in the returned - kv_transfer_params so the D worker can surface them in - prompt_tokens_details instead of the ~100% local hit it measures - when pulling the KVs from the remote. - """ - vllm_config = create_vllm_config() - scheduler = create_scheduler(vllm_config) - - BLOCK_SIZE = vllm_config.cache_config.block_size - NUM_TOKENS = BLOCK_SIZE * 3 - - # Warm the prefix cache with a request sharing the full prompt. - warmup = create_request( - request_id=1, - num_tokens=NUM_TOKENS, - common_prefix_len=NUM_TOKENS, - block_size=BLOCK_SIZE, - ) - scheduler.add_request(warmup) - scheduler_output = scheduler.schedule() - model_runner_output = create_model_runner_output([warmup], use_eos=True) - scheduler.update_from_output(scheduler_output, model_runner_output) - - # P-side request with the same prompt hits the local cache on all but - # the last block (recomputed to obtain logits). - request = create_request( - request_id=2, - num_tokens=NUM_TOKENS, - common_prefix_len=NUM_TOKENS, - block_size=BLOCK_SIZE, - do_remote_decode=True, - ) - scheduler.add_request(request) - scheduler_output = scheduler.schedule() - assert request.prefill_stats is not None - assert request.prefill_stats.num_cached_tokens == NUM_TOKENS - BLOCK_SIZE - - # max_tokens=1, so the request finishes and returns kv_transfer_params. - model_runner_output = create_model_runner_output([request]) - engine_core_outputs = scheduler.update_from_output( - scheduler_output, model_runner_output - ) - - output = engine_core_outputs[0].outputs[0] - assert output.finish_reason is not None - assert output.kv_transfer_params is not None - assert ( - output.kv_transfer_params["remote_prefill_cached_tokens"] - == NUM_TOKENS - BLOCK_SIZE - ) - - -@patch( - "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper", - FakeNixlWrapper, -) -def test_kv_transfer_handshake(dist_init): - """Unit test for basic NixlConnector interface functionality.""" - # Test setup, we creates a scheduler that contains a NixlConnector - # of role SCHEDULER, and expect it to be serving NixlAgentMetadata from - # all workers of the instance. - block_size = maybe_update_block_size(16) - vllm_config = create_vllm_config(block_size=block_size) - vllm_config.cache_config.kv_cache_layout = "BLHNC" - # in case the test runs on non-GPU machine - vllm_config.kv_transfer_config.kv_buffer_device = "cpu" - scheduler = create_scheduler(vllm_config) - - with set_current_vllm_config(vllm_config): - # Create two NixlConnector of role WORKER, one is the worker of - # the scheduler (prefill), the other is a worker of decode instance. - - # Prefill connector will register KV cache to populate proper handshake - # metadata. - kv_cache_groups = [ - KVCacheGroupSpec( - ["layer0", "layer1", "layer2"], - FullAttentionSpec( - block_size=block_size, - num_kv_heads=4, - head_size=16, - dtype=torch.float16, - ), - ) - ] - kv_cache_config = KVCacheConfig( - num_blocks=2, kv_cache_tensors=[], kv_cache_groups=kv_cache_groups - ) - prefill_connector = NixlConnector( - vllm_config, KVConnectorRole.WORKER, kv_cache_config - ) - kv_cache_spec = cast( - AttentionSpec, kv_cache_config.kv_cache_groups[0].kv_cache_spec - ) - raw = torch.zeros( - kv_cache_spec.page_size_bytes * kv_cache_config.num_blocks * 3, - dtype=torch.int8, - ) - caches = dense_kv_cache_views( - raw, - kv_cache_spec, - kv_cache_config.num_blocks, - num_layers=3, - layout=KVCacheLayout.BLHNC, - ) - kv_caches = { - f"layer{layer_idx}": cache for layer_idx, cache in enumerate(caches) - } - prefill_connector.register_kv_caches(kv_caches) - - # Simulate EngineCore initialization that would gather connector - # metadata from all workers - metadata = prefill_connector.get_handshake_metadata() - - # metadata is a NixlHandshakePayload, decode it to get NixlAgentMetadata - decoder = msgspec.msgpack.Decoder(NixlAgentMetadata) - expected_agent_metadata = decoder.decode(metadata.agent_metadata_bytes) - - # The scheduler connector expects metadata keyed by - # (pp_rank, tp_rank). - scheduler_connector = scheduler.get_kv_connector() - scheduler_connector.set_xfer_handshake_metadata_pp_aware({(0, 0): metadata}) - - # Simulate a request that finishes prefill, which returns - # corresponding NixlConnectorMetadata for decode instance. - BLOCK_SIZE = vllm_config.cache_config.block_size - NUM_EXTERNAL_FULL_BLOCKS = 2 - NUM_TOKENS = int(BLOCK_SIZE * (NUM_EXTERNAL_FULL_BLOCKS + 0.5)) - - request = create_request( - request_id=1, - block_size=BLOCK_SIZE, - num_tokens=NUM_TOKENS, - do_remote_decode=True, - ) - request.status = RequestStatus.FINISHED_LENGTH_CAPPED - delay, kv_connector_metadata = ( - scheduler.get_kv_connector().request_finished_all_groups( - request, ([0, 1, 2],) - ) - ) - assert delay - # Pull connector advertises its transfer mode in kv_transfer_params so - # an external router can distinguish it from a push producer. - assert kv_connector_metadata["transfer_mode"] == "pull" - - # Decode connector will be able to create handshake with the prefill connector. - decode_connector = NixlConnector( - vllm_config, KVConnectorRole.WORKER, kv_cache_config - ) - decode_connector.register_kv_caches(kv_caches) - - # Here we are testing the retrieval of NIXLAgentMetadata. - # Knowing the implementation detail, we override the add_remote_agent - # to validate the metadata received is the same as the one in prefill_connector. - with patch.object( - decode_connector.connector_worker, "add_remote_agent" - ) as mock_add_remote_agent: - mock_add_remote_agent.return_type = "remote_agent" - - decode_connector.connector_worker._nixl_handshake( - kv_connector_metadata["remote_host"], - kv_connector_metadata["remote_port"], - kv_connector_metadata["tp_size"], - kv_connector_metadata["remote_engine_id"], - ) - - received_metadata = mock_add_remote_agent.call_args.args - assert received_metadata[0] == expected_agent_metadata - assert received_metadata[1] == 0 # remote_tp_rank - assert received_metadata[2] == 1 # remote_tp_size - - # Need to shutdown the background thread to release NIXL side channel port - scheduler_connector.shutdown() - - -class FakeNixlConnectorWorker(NixlConnectorWorker): - REMOTE_ENGINE_ID = "remote_engine" - - def __init__( - self, - *args, - hand_shake_latency: float = 1.8, - kv_cache_layout="LBHNC", - kv_cache_config=None, - **kwargs, - ): - if kv_cache_config is None: - kv_cache_config = make_kv_cache_config( - block_size=args[0].cache_config.block_size - ) - super().__init__(*args, kv_cache_config=kv_cache_config, **kwargs) - self._hand_shake_latency = hand_shake_latency - self.kv_cache_layout = kv_cache_layout - # Mock register_kv_caches attributes needed for tests that do not call it. - self.src_xfer_handles_by_block_size = {self.block_size: 1} - self.src_blocks_data = np.empty((0, 5), dtype=np.uint64) - rep_spec = self.kv_cache_config.kv_cache_groups[0].kv_cache_spec - test_shape = compute_layer_kv_cache_shape_bytes(rep_spec, 1) - self.transfer_topo = TransferTopology( - tp_rank=self.tp_rank, - tp_size=self.world_size, - block_size=self.block_size, - engine_id=self.engine_id, - is_mla=self.use_mla, - is_mamba=False, - total_num_kv_heads=self.model_config.get_total_num_kv_heads(), - attn_backends=self.attn_backends, - tensor_shape=test_shape, - ) - - self.compat_hash = compute_nixl_compatibility_hash( - self.vllm_config, self.backend_name - ) - - def _nixl_handshake( - self, - host: str, - port: int, - remote_tp_size: int, - expected_engine_id: str, - remote_dcp_size: int = 1, - remote_pp_size: int = 1, - notif_agents_only: bool = False, - ) -> tuple[dict[tuple[int, int], str], float]: - # Mimic slow _nixl_handshake, as well as bypass zmq communication. - time.sleep(self._hand_shake_latency) - # These should've been done in register_kv_caches(), called by - # gpu_model_runner. Here we just hardcode some dummy values. - slot_size_bytes = 4096 - self.slot_size_per_layer = [slot_size_bytes] - self.block_len_per_layer = [slot_size_bytes * self.block_size] - self.num_blocks = self.kv_cache_config.num_blocks - self.num_regions = 1 - self.block_stride_per_layer = list(self.block_len_per_layer) - self.region_num_blocks = [self.num_blocks] - self.region_group_ids = [0] - self.region_mem_types = [self.nixl_memory_type] - self.dst_num_blocks[self.engine_id] = self.num_blocks - self.dst_region_num_blocks[self.engine_id] = self.region_num_blocks - self.dst_region_group_ids[self.engine_id] = self.region_group_ids - self.dst_region_mem_types[self.engine_id] = self.region_mem_types - - assert expected_engine_id == self.REMOTE_ENGINE_ID - - # Adjust remote block length metadata to satisfy heterogeneous TP - # invariants enforced during handshake validation. Use per-rank - # head ratio (not tp_ratio) to account for GQA replication capping. - remote_block_lens = list(self.block_len_per_layer) - tp_ratio = self.transfer_topo.tp_ratio(remote_tp_size) - total_kv = self.transfer_topo.total_num_kv_heads - local_heads = self.transfer_topo.local_physical_heads - remote_heads = max(1, total_kv // remote_tp_size) - if remote_tp_size != self.world_size: - remote_block_lens = [ - block_len * remote_heads // local_heads - for block_len in remote_block_lens - ] - - # When remote tp_size > local tp_size, handshake with multiple - # remote ranks. - num_handshakes = 1 if tp_ratio > 0 else -tp_ratio - remote_agents: dict[tuple[int, int], str] = {} - for remote_tp_rank in range(num_handshakes): - remote_agent_name = self.add_remote_agent( - NixlAgentMetadata( - engine_id=self.REMOTE_ENGINE_ID, - agent_metadata=FakeNixlWrapper.AGENT_METADATA, - kv_caches_base_addr=[0], - device_id=remote_tp_rank, - num_blocks=self.num_blocks, - block_lens=remote_block_lens, - block_strides=remote_block_lens, - kv_cache_layout="LBHNC", - block_size=self.block_size, - ssm_sizes=(0, 0), - attn_backend_name=self.backend_name, - physical_blocks_per_logical_kv_block=1, - region_num_blocks=self.region_num_blocks, - region_group_ids=self.region_group_ids, - region_mem_types=self.region_mem_types, - ), - remote_tp_rank=remote_tp_rank, - remote_tp_size=remote_tp_size, - ) - remote_agents[(0, remote_tp_rank)] = remote_agent_name - # Handshake bypasses zmq, so report a zero clock offset to the peer. - return remote_agents, 0.0 - - -class TestNixlHandshake: - @pytest.mark.parametrize( - ("pcp_rank", "pcp_size", "dcp_size", "expected_tracked"), - [ - (0, 2, 1, True), - (1, 2, 1, False), - (0, 2, 2, True), - (1, 2, 2, True), - (0, 4, 4, True), - (1, 4, 4, True), - (2, 4, 4, True), - (3, 4, 4, True), - ], - ) - @patch( - "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper", - FakeNixlWrapper, - ) - def test_pcp_producer_exposes_dcp_shards_or_canonical_replica( - self, - default_vllm_config, - dist_init, - pcp_rank, - pcp_size, - dcp_size, - expected_tracked, - ): - """Replicated PCP is canonicalized; PCP-DCP publishes every shard.""" - from vllm.v1.attention.backends.flash_attn import FlashAttentionBackend - - block_size = maybe_update_block_size(16) - vllm_config = create_vllm_config(kv_role="kv_producer", block_size=block_size) - vllm_config.parallel_config.prefill_context_parallel_size = pcp_size - vllm_config.parallel_config.decode_context_parallel_size = dcp_size - with ( - patch( - "vllm.distributed.kv_transfer.kv_connector.v1.nixl." - "base_worker.get_current_attn_backends", - return_value=[FlashAttentionBackend], - ), - patch( - "vllm.distributed.kv_transfer.kv_connector.v1.nixl." - "base_worker.get_pcp_group" - ) as mock_get_pcp_group, - ): - mock_get_pcp_group.return_value.rank_in_group = pcp_rank - connector = NixlConnector( - vllm_config, - KVConnectorRole.WORKER, - make_kv_cache_config(block_size=block_size), - ) - - worker = connector.connector_worker - assert worker is not None - assert worker.pcp_rank == pcp_rank - assert worker.pcp_dcp_sharded is (dcp_size > 1) - assert worker.transfer_tp_rank == (pcp_rank if dcp_size > 1 else 0) - assert worker.transfer_tp_size == (pcp_size if dcp_size > 1 else 1) - - req_id = "req" - metadata = NixlConnectorMetadata() - metadata.reqs_in_batch.add(req_id) - metadata.reqs_to_send[req_id] = time.perf_counter() + 10 - worker.start_load_kv(metadata) - assert (req_id in worker._reqs_to_process) == expected_tracked - assert (req_id in worker._reqs_to_send) == expected_tracked - - payload = MagicMock(spec=NixlHandshakePayload) - worker.xfer_handshake_metadata = payload - worker.transfer_topo = MagicMock() - worker._get_new_notifs = MagicMock( - side_effect=lambda: {"sent"} if expected_tracked else set() - ) - - expected_payload = payload if expected_tracked else None - assert connector.get_handshake_metadata() is expected_payload - done_sending, done_recving = connector.get_finished(set()) - assert done_sending == ({"sent"} if expected_tracked else {req_id}) - assert done_recving == set() - if not expected_tracked: - assert connector.get_finished(set()) == (set(), set()) - - worker.get_transfer_results = MagicMock( - return_value=KVConnectorTransferResults(finished_sending={"sent"}) - ) - results = connector.get_transfer_results(set()) - assert results.finished_sending == ({"sent"} if expected_tracked else set()) - - @patch( - "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper", - FakeNixlWrapper, - ) - def test_multi_xfer_one_engine( - self, - default_vllm_config, - # dist_init is a fixture that initializes the distributed environment. - dist_init, - ): - """Test case where multiple xfers are initiated to the same engine. - - This test triggers the connector to load remote KV for the same - `request_id`. - """ - block_size = maybe_update_block_size(16) - vllm_config = create_vllm_config(block_size=block_size) - - request_id = "req_id" - - # Test worker role in decode server. - kv_cache_config = make_kv_cache_config(block_size=block_size, num_blocks=10) - connector = NixlConnector(vllm_config, KVConnectorRole.WORKER, kv_cache_config) - connector.connector_worker = FakeNixlConnectorWorker( - vllm_config, - connector.engine_id, - hand_shake_latency=0, - kv_cache_config=kv_cache_config, - ) - assert isinstance(connector.connector_worker.nixl_wrapper, FakeNixlWrapper) - worker = connector.connector_worker - # simulate handshake - worker.dst_xfer_side_handles = { - FakeNixlConnectorWorker.REMOTE_ENGINE_ID: {0: 1} - } - worker.kv_cache_layout = "LBHNC" - num_xfers = 4 - while True: - # For the same request_id, initiate multiple xfers across different - # round of `execute_model` calls. - metadata = NixlConnectorMetadata() - if num_xfers > 0: - num_xfers -= 1 - metadata.add_new_req_to_recv( - request_id=request_id, - local_block_ids=([num_xfers + 1, num_xfers + 2, num_xfers + 3],), - kv_transfer_params={ - "remote_block_ids": ( - [ - num_xfers + 4, - num_xfers + 5, - num_xfers + 6, - ], - ), - "remote_engine_id": FakeNixlConnectorWorker.REMOTE_ENGINE_ID, - "remote_request_id": f"prefill-{request_id}", - "remote_host": "localhost", - "remote_port": 1234, - "remote_tp_size": 1, - }, - ) - connector.bind_connector_metadata(metadata) - - # Mimic logic in KVConnectorModelRunnerMixin._get_kv_connector_output. - dummy_ctx = ForwardContext( - no_compile_layers={}, - attn_metadata={}, - slot_mapping={}, - ) - _before_load = time.perf_counter() - connector.start_load_kv(dummy_ctx) - _after_load = time.perf_counter() - assert _after_load - _before_load < 0.1, ( - f"start_load_kv took {_after_load - _before_load} seconds" - ) - - # Mimic logic in KVConnectorModelRunnerMixin._get_kv_connector_output. - _, done_recving = connector.get_finished(finished_req_ids=set()) - if len(done_recving) > 0: - assert request_id in done_recving - break - - connector.clear_connector_metadata() - - @patch( - "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper", - FakeNixlWrapper, - ) - @pytest.mark.parametrize( - "decode_tp_size, prefill_tp_size", - [ - (1, 1), - (2, 1), - (4, 2), - (4, 4), - ], - ) - def test_async_load_kv( - self, - default_vllm_config, - # Fixture that initializes the distributed environment. - dist_init, - # Simulate consumer-producer TP sizes. - decode_tp_size, - prefill_tp_size, - ): - """Test that NixlConnector's start_load_kv should be non-blocking.""" - block_size = maybe_update_block_size(16) - vllm_config = create_vllm_config(block_size=block_size) - vllm_config.parallel_config.tensor_parallel_size = decode_tp_size - - # Test worker role in decode server. - connector = NixlConnector( - vllm_config, - KVConnectorRole.WORKER, - make_kv_cache_config(block_size=block_size), - ) - connector.connector_worker = FakeNixlConnectorWorker( - vllm_config, connector.engine_id - ) - metadata = NixlConnectorMetadata() - metadata.add_new_req_to_recv( - request_id="id", - local_block_ids=([1, 2, 3],), - kv_transfer_params={ - "remote_block_ids": ([4, 5, 6],), - "remote_engine_id": FakeNixlConnectorWorker.REMOTE_ENGINE_ID, - "remote_request_id": "prefill-id", - "remote_host": "localhost", - "remote_port": 1234, - "remote_tp_size": prefill_tp_size, - }, - ) - connector.bind_connector_metadata(metadata) - - timeout = 2.5 - start = time.perf_counter() - while time.perf_counter() - start < timeout: - dummy_ctx = ForwardContext( - no_compile_layers={}, - attn_metadata={}, - slot_mapping={}, - ) - _before_load = time.perf_counter() - connector.start_load_kv(dummy_ctx) - _after_load = time.perf_counter() - assert _after_load - _before_load < 0.1, ( - f"start_load_kv took {_after_load - _before_load} seconds" - ) - time.sleep(0.5) # backoff for the async handshake to complete. - connector.bind_connector_metadata(NixlConnectorMetadata()) - _, done_recving = connector.get_finished(finished_req_ids=set()) - if len(done_recving) > 0: - return - raise TimeoutError("Took too long to complete async handshake.") - - @patch( - "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper", - FakeNixlWrapper, - ) - @pytest.mark.parametrize("local_tp_size", [1, 2]) - def test_prefill_tp_size_greater_than_decode_tp_size( - self, local_tp_size: int, default_vllm_config, dist_init, monkeypatch - ): - """Verify remote TP > local TP handshake succeeds with different - remote configurations. - """ - monkeypatch.setattr( - "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.get_tensor_model_parallel_world_size", - lambda: local_tp_size, - ) - - block_size = maybe_update_block_size(16) - vllm_config = create_vllm_config(block_size=block_size) - - # num_blocks=1 keeps the fake handshake's region_num_blocks consistent - # with the minimal single-block registration below (num_descs == 1); - # _fa_desc_replicated cross-checks the two. - kv_cache_config = make_kv_cache_config(block_size=block_size, num_blocks=1) - connector = NixlConnector(vllm_config, KVConnectorRole.WORKER, kv_cache_config) - connector.connector_worker = FakeNixlConnectorWorker( - vllm_config, - connector.engine_id, - hand_shake_latency=0, - kv_cache_config=kv_cache_config, - ) - worker = connector.connector_worker - - # Minimal local registration params used by add_remote_agent - worker.slot_size_per_layer = [4096] - worker.block_len_per_layer = [4096 * worker.block_size] - worker.num_blocks = 1 - worker.dst_num_blocks[worker.engine_id] = worker.num_blocks - block_len = worker.block_len_per_layer[0] - worker.src_blocks_data = np.array( - [(0, block_len, worker.tp_rank, block_len, worker.num_blocks)], - dtype=np.uint64, - ) - worker.num_descs = int(worker.src_blocks_data[:, 4].sum()) - - def check_handshake(remote_tp_size: int): - tp_ratio = remote_tp_size // local_tp_size - assert set(remote_agents.keys()) == {(0, r) for r in range(tp_ratio)} - - remote_engine_id = worker.REMOTE_ENGINE_ID - remote_info = worker.transfer_topo.get_engine_info(remote_engine_id) - assert remote_info.remote_tp_size == remote_tp_size - assert -tp_ratio == worker.transfer_topo.tp_ratio(remote_tp_size) - # ensure src_xfer_handles_by_tp_ratio is populated with tpratio chunks - split_key = (-tp_ratio, worker.block_size) - assert split_key in worker.src_xfer_handles_by_tp_ratio - assert len(worker.src_xfer_handles_by_tp_ratio[split_key]) == tp_ratio - assert remote_engine_id in worker.dst_xfer_side_handles - assert set(worker.dst_xfer_side_handles[remote_engine_id].keys()) == set( - range(tp_ratio) - ) - - remote_agents, _ = worker._nixl_handshake( - host="localhost", - port=1234, - remote_tp_size=4, - expected_engine_id=worker.REMOTE_ENGINE_ID, - ) - check_handshake(4) - - # NOTE flexibility: a second remote with higher number of ranks is - # discovered. This is not a scenario we actively support right now, but - # the connector allows it. - worker.REMOTE_ENGINE_ID = "remote_engine_2" - remote_agents, _ = worker._nixl_handshake( - host="localhost", - port=1234, - remote_tp_size=6, - expected_engine_id=worker.REMOTE_ENGINE_ID, - ) - check_handshake(6) - - @patch( - "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper", - FakeNixlWrapper, - ) - def test_prefill_tp_size_greater_than_decode_tp_size_mla( - self, default_vllm_config, dist_init - ): - """Verify remote TP > local TP handshake succeeds with different - remote configurations for an MLA model. - """ - block_size = maybe_update_block_size(16) - vllm_config = create_vllm_config(block_size=block_size) - d_tp_size = 1 - p_tp_size = 2 - - # Build two separate connectors/workers to emulate P TP=2 ranks. - conn_p0 = NixlConnector( - vllm_config, - KVConnectorRole.WORKER, - make_kv_cache_config(block_size=block_size), - ) - conn_p1 = NixlConnector( - vllm_config, - KVConnectorRole.WORKER, - make_kv_cache_config(block_size=block_size), - ) - conn_p0.connector_worker = FakeNixlConnectorWorker( - vllm_config, conn_p0.engine_id, hand_shake_latency=0 - ) - conn_p1.connector_worker = FakeNixlConnectorWorker( - vllm_config, conn_p1.engine_id, hand_shake_latency=0 - ) - - # Force P world size to 2 for both workers and emulate distinct tp_ranks. - # Also enable MLA path so that expected_finished_count is updated. - for rank, worker in enumerate( - (conn_p0.connector_worker, conn_p1.connector_worker) - ): - worker.world_size = p_tp_size - worker.transfer_topo.tp_size = p_tp_size - worker.tp_rank = rank - worker.use_mla = True - - req_id = "req-ep-dp2-p0" - now = time.perf_counter() - # Register a request on P that is waiting for consumers to read - # (both workers track it). - conn_p0.connector_worker._reqs_to_send[req_id] = now + 10.0 - conn_p0.connector_worker._reqs_to_process.add(req_id) - conn_p1.connector_worker._reqs_to_send[req_id] = now + 10.0 - conn_p1.connector_worker._reqs_to_process.add(req_id) - - # Simulate a read notification coming from D with (tp=1, dp=2). - notif = f"{req_id}:{d_tp_size}".encode() - # D0-0->P0 notif - conn_p0.connector_worker.nixl_wrapper.get_new_notifs = lambda: { - "agent": [notif] - } # type: ignore[method-assign] - conn_p1.connector_worker.nixl_wrapper.get_new_notifs = lambda: { - "agent": [notif] - } # type: ignore[method-assign] - - # Trigger notification processing via get_finished(). - done_sending0, _ = conn_p0.get_finished(finished_req_ids=set()) - done_sending1, _ = conn_p1.get_finished(finished_req_ids=set()) - assert req_id in done_sending0 and req_id in done_sending1 - - # E2E aggregation: ensure the aggregated output marks the request - # as finished using the connector's expected_finished_count. - from vllm.v1.outputs import KVConnectorOutput, ModelRunnerOutput - - aggregator = KVOutputAggregator.from_connector(conn_p0, world_size=2) - - out0 = ModelRunnerOutput( - req_ids=[req_id], - req_id_to_index={req_id: 0}, - sampled_token_ids=[[0]], - logprobs=None, - prompt_logprobs_dict={}, - pooler_output=[None], - kv_connector_output=KVConnectorOutput( - finished_sending=done_sending0, - finished_recving=None, - ), - ) - out1 = ModelRunnerOutput( - req_ids=[req_id], - req_id_to_index={req_id: 0}, - sampled_token_ids=[[0]], - logprobs=None, - prompt_logprobs_dict={}, - pooler_output=[None], - kv_connector_output=KVConnectorOutput( - finished_sending=done_sending1, - finished_recving=None, - ), - ) - aggregated = aggregator.aggregate([out0, out1], output_rank=0) - assert aggregated.kv_connector_output is not None - assert aggregated.kv_connector_output.finished_sending == {req_id} - - # Producers cleaned up state for the finished request. - assert req_id not in conn_p0.connector_worker._reqs_to_send - assert req_id not in conn_p0.connector_worker._reqs_to_process - assert req_id not in conn_p1.connector_worker._reqs_to_send - assert req_id not in conn_p1.connector_worker._reqs_to_process - - @patch( - "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper", - FakeNixlWrapper, - ) - def test_concurrent_load_kv( - self, - default_vllm_config, - # dist_init is a fixture that initializes the distributed environment. - dist_init, - ): - """Test that multiple start_load_kv calls should occur concurrently.""" - block_size = maybe_update_block_size(16) - vllm_config = create_vllm_config(block_size=block_size) - - # Test worker role in decode server. - connector = NixlConnector( - vllm_config, - KVConnectorRole.WORKER, - make_kv_cache_config(block_size=block_size), - ) - connector.connector_worker = FakeNixlConnectorWorker( - vllm_config, connector.engine_id - ) - # Register (mocked) local xfer handler - # worker = connector.connector_worker - # worker.src_xfer_handles_by_block_size = {worker.block_size: 1} - metadata = NixlConnectorMetadata() - total_reqs = 5 - for i in range(total_reqs): - metadata.add_new_req_to_recv( - request_id=f"id_{i}", - local_block_ids=([1, 2, 3],), - kv_transfer_params={ - "remote_block_ids": ([4, 5, 6],), - "remote_engine_id": FakeNixlConnectorWorker.REMOTE_ENGINE_ID, - "remote_request_id": f"prefill-id-{i}", - "remote_host": "localhost", - "remote_port": 1234, - "remote_tp_size": 1, - }, - ) - connector.bind_connector_metadata(metadata) - - timeout = 2.5 * total_reqs - cnt_finished_reqs = 0 - start = time.perf_counter() - while time.perf_counter() - start < timeout: - dummy_ctx = ForwardContext( - no_compile_layers={}, - attn_metadata={}, - slot_mapping={}, - ) - _before_load = time.perf_counter() - connector.start_load_kv(dummy_ctx) - _after_load = time.perf_counter() - assert _after_load - _before_load < 0.1, ( - f"start_load_kv took {_after_load - _before_load} seconds" - ) - time.sleep(0.5) # backoff for the async handshake to complete. - connector.bind_connector_metadata(NixlConnectorMetadata()) - _, done_recving = connector.get_finished(finished_req_ids=set()) - if len(done_recving) > 0: - cnt_finished_reqs += len(done_recving) - if cnt_finished_reqs == total_reqs: - return - raise TimeoutError("Took too long to complete async handshake.") - - @pytest.mark.parametrize( - ("kv_cache_layout", "enable_permute_local_kv"), - # enable_permute_local_kv only covers a local LBNHC cache. - [("LBHNC", False), ("BLHNC", True)], - ) - @patch( - "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper", - FakeNixlWrapper, - ) - def test_handshake_fails_on_kv_cache_layout_mismatch( - self, default_vllm_config, dist_init, kv_cache_layout, enable_permute_local_kv - ): - """Verify that adding a remote agent fails if kv_cache_layout differs. - This test is only relevant for heterogeneous TP. - """ - block_size = maybe_update_block_size(16) - vllm_config = create_vllm_config( - enable_permute_local_kv=enable_permute_local_kv, - block_size=block_size, - ) - - # Mock TP world size to 2 to force heterogeneous TP when - # remote_tp_size=1 - with patch( - "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.get_tensor_model_parallel_world_size", # noqa: E501 - return_value=2, - ): - # Initialize connector and worker (with fake NIXL wrapper) - connector = NixlConnector( - vllm_config, - KVConnectorRole.WORKER, - make_kv_cache_config(block_size=block_size), - ) - connector.connector_worker = FakeNixlConnectorWorker( - vllm_config, - connector.engine_id, - hand_shake_latency=0, - kv_cache_layout=kv_cache_layout, - ) - worker = connector.connector_worker - - # Minimal local registration params used by add_remote_agent - worker.slot_size_per_layer = [4096] - worker.block_len_per_layer = [4096 * worker.block_size] - worker.num_blocks = 1 - worker.dst_num_blocks[worker.engine_id] = worker.num_blocks - - # Metadata with different kv_cache_layout than local worker - mismatched_layout = ( - "LBHNC" if worker.kv_cache_layout != "LBHNC" else "LBNHC" - ) - meta = NixlAgentMetadata( - engine_id=FakeNixlConnectorWorker.REMOTE_ENGINE_ID, - agent_metadata=FakeNixlWrapper.AGENT_METADATA, - kv_caches_base_addr=[0], - device_id=0, - num_blocks=1, - block_lens=worker.block_len_per_layer, - block_strides=worker.block_len_per_layer, - kv_cache_layout=mismatched_layout, - block_size=worker.block_size, - ssm_sizes=(0, 0), - attn_backend_name=worker.backend_name, - physical_blocks_per_logical_kv_block=1, - ) - - with pytest.raises(RuntimeError): - # mismatched layout is expected to fail - worker.add_remote_agent(meta, remote_tp_size=2) - worker.add_remote_agent(meta, remote_tp_size=1) - - @patch( - "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper", - FakeNixlWrapper, - ) - def test_handshake_succeed_on_kv_cache_layout_mismatch_with_experimental( - self, default_vllm_config, dist_init - ): - """Verify that adding a remote agent fails if kv_cache_layout differs. - This test is only relevant for heterogeneous TP. - """ - block_size = maybe_update_block_size(16) - vllm_config = create_vllm_config( - enable_permute_local_kv=True, block_size=block_size - ) - - # Mock TP world size to 2 to force heterogeneous TP when - # remote_tp_size=1 - with patch( - "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.get_tensor_model_parallel_world_size", # noqa: E501 - return_value=2, - ): - # Initialize connector and worker (with fake NIXL wrapper) - connector = NixlConnector( - vllm_config, - KVConnectorRole.WORKER, - make_kv_cache_config(block_size=block_size), - ) - connector.connector_worker = FakeNixlConnectorWorker( - vllm_config, - connector.engine_id, - hand_shake_latency=0, - kv_cache_layout="LBNHC", - ) - worker = connector.connector_worker - - # Minimal local registration params used by add_remote_agent - worker.slot_size_per_layer = [2048] - worker.block_len_per_layer = [2048 * worker.block_size] - worker.num_blocks = 1 - worker.dst_num_blocks[worker.engine_id] = worker.num_blocks - - # Metadata with different kv_cache_layout than local worker - # prefill TP=1, decode TP=2, remote block_lens is double to local - remote_block_lens = [i * 2 for i in worker.block_len_per_layer] - meta = NixlAgentMetadata( - engine_id=FakeNixlConnectorWorker.REMOTE_ENGINE_ID, - agent_metadata=FakeNixlWrapper.AGENT_METADATA, - kv_caches_base_addr=[0], - device_id=0, - num_blocks=1, - block_lens=remote_block_lens, - block_strides=remote_block_lens, - kv_cache_layout="LBHNC", - block_size=worker.block_size, - ssm_sizes=(0, 0), - attn_backend_name=worker.backend_name, - physical_blocks_per_logical_kv_block=1, - ) - - # We don't check layout for homogeneous TP and MLA for now, as the - # whole block is moved. - worker.add_remote_agent(meta, remote_tp_size=1) - - @patch( - "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper", - FakeNixlWrapper, - ) - def test_hybrid_mamba_attention_remote_descs_use_packed_head_slices( - self, default_vllm_config, dist_init - ): - worker = FakeNixlConnectorWorker( - create_vllm_config(block_size=maybe_update_block_size(16)), - "engine", - hand_shake_latency=0, - ) - - remote_block_len = 2048 - local_block_len = remote_block_len // 2 - worker.block_len_per_layer = [local_block_len] - worker._region_is_mla = [False] - worker.num_blocks = 1 - worker.num_regions = 1 - worker._has_mamba = True - worker._mamba_ssm_size = (128, 256) - worker.transfer_topo = TransferTopology( - tp_rank=1, - tp_size=2, - block_size=worker.block_size, - engine_id=worker.engine_id, - is_mla=False, - is_mamba=True, - total_num_kv_heads=2, - attn_backends=worker.attn_backends, - tensor_shape=None, - ) - plan = MagicMock( - source_ranks_per_group=((0,), (0,)), - rank_offset_factor=1, - ) - meta = NixlAgentMetadata( - engine_id=FakeNixlConnectorWorker.REMOTE_ENGINE_ID, - agent_metadata=FakeNixlWrapper.AGENT_METADATA, - kv_caches_base_addr=[0x1000], - device_id=0, - num_blocks=1, - block_lens=[remote_block_len], - block_strides=[remote_block_len], - kv_cache_layout="HND", - block_size=worker.block_size, - ssm_sizes=(0, 0), - attn_backend_name=worker.backend_name, - physical_blocks_per_logical_kv_block=1, - region_num_blocks=None, - ) - - assert worker._build_fa_remote(plan, meta, block_size_ratio=1).tolist() == [ - [0x1000 + local_block_len, local_block_len, 0, remote_block_len, 1] - ] - - @patch( - "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper", - FakeNixlWrapper, - ) - def test_handshake_mixed_fa_mla_hetero_tp(self, default_vllm_config, dist_init): - """Mixed full-attn (SPLIT) + MLA (REPLICATE) single KV group under - heterogeneous TP must NOT raise (previously a NotImplementedError), - and the per-region gate must still reject a wrong block_len. - """ - block_size = maybe_update_block_size(16) - vllm_config = create_vllm_config(block_size=block_size) - with patch( - "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.get_tensor_model_parallel_world_size", # noqa: E501 - return_value=2, - ): - connector = NixlConnector( - vllm_config, - KVConnectorRole.WORKER, - make_kv_cache_config(block_size=block_size), - ) - connector.connector_worker = FakeNixlConnectorWorker( - vllm_config, connector.engine_id, hand_shake_latency=0 - ) - worker = connector.connector_worker - - # Region 0: full-attn (SPLIT). Region 1: MLA (REPLICATE). - fa_len = 4096 * worker.block_size - idx_len = 512 * worker.block_size - worker.slot_size_per_layer = [4096, 512] - worker.block_len_per_layer = [fa_len, idx_len] - worker._region_is_mla = [False, True] - worker.num_blocks = 1 - worker.dst_num_blocks[worker.engine_id] = worker.num_blocks - worker.src_blocks_data = np.array( - [ - (0, fa_len, worker.tp_rank, fa_len, worker.num_blocks), - (0, idx_len, worker.tp_rank, idx_len, worker.num_blocks), - ], - dtype=np.uint64, - ) - worker.num_descs = int(worker.src_blocks_data[:, 4].sum()) - - # D_TP=2, P_TP=1 -> tp_ratio=2. SPLIT region scales by tp_ratio; - # REPLICATE region is unchanged. - tp_ratio = 2 - meta = NixlAgentMetadata( - engine_id=FakeNixlConnectorWorker.REMOTE_ENGINE_ID, - agent_metadata=FakeNixlWrapper.AGENT_METADATA, - kv_caches_base_addr=[0, 0], - device_id=0, - num_blocks=1, - block_lens=[fa_len * tp_ratio, idx_len], - block_strides=[fa_len * tp_ratio, idx_len], - kv_cache_layout=worker.kv_cache_layout, - block_size=worker.block_size, - ssm_sizes=(0, 0), - attn_backend_name=worker.backend_name, - physical_blocks_per_logical_kv_block=1, - ) - worker.add_remote_agent(meta, remote_tp_size=1) - assert ( - FakeNixlConnectorWorker.REMOTE_ENGINE_ID in worker.dst_xfer_side_handles - ) - # Gate rejects an MLA region wrongly scaled by tp_ratio. - worker2 = FakeNixlConnectorWorker( - vllm_config, connector.engine_id, hand_shake_latency=0 - ) - worker2.block_len_per_layer = [fa_len, idx_len] - worker2._region_is_mla = [False, True] - worker2.num_blocks = 1 - worker2.dst_num_blocks[worker2.engine_id] = worker2.num_blocks - bad_meta = NixlAgentMetadata( - engine_id=FakeNixlConnectorWorker.REMOTE_ENGINE_ID, - agent_metadata=FakeNixlWrapper.AGENT_METADATA, - kv_caches_base_addr=[0, 0], - device_id=0, - num_blocks=1, - # WRONG: MLA region scaled by tp_ratio (it should be replicated). - block_lens=[fa_len * tp_ratio, idx_len * tp_ratio], - block_strides=[fa_len * tp_ratio, idx_len * tp_ratio], - kv_cache_layout=worker2.kv_cache_layout, - block_size=worker2.block_size, - ssm_sizes=(0, 0), - attn_backend_name=worker2.backend_name, - physical_blocks_per_logical_kv_block=1, - ) - with pytest.raises(AssertionError): - worker2.add_remote_agent(bad_meta, remote_tp_size=1) - - @patch( - "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper", - FakeNixlWrapper, - ) - def test_handshake_validates_gqa_replicated_block_len( - self, default_vllm_config, dist_init - ): - """Regression test for #45330. - - When tp_size > total_num_kv_heads, GQA replication caps per-rank - KV heads at 1, so block_len stops scaling with 1/tp. With 8 KV - heads and D_TP=16 pulling from P_TP=8, both sides hold one head - per rank and report the *same* block_len; the old validation - expected local_block_len * tp_ratio and rejected the valid - handshake. - """ - block_size = maybe_update_block_size(16) - vllm_config = create_vllm_config(block_size=block_size) - - with patch( - "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.get_tensor_model_parallel_world_size", # noqa: E501 - return_value=16, - ): - connector = NixlConnector( - vllm_config, - KVConnectorRole.WORKER, - make_kv_cache_config(block_size=block_size), - ) - connector.connector_worker = FakeNixlConnectorWorker( - vllm_config, connector.engine_id, hand_shake_latency=0 - ) - worker = connector.connector_worker - - worker.transfer_topo.total_num_kv_heads = 8 - worker.transfer_topo.local_physical_heads = 1 - worker.kv_cache_layout = "LBHNC" - - worker.slot_size_per_layer = [4096] - worker.block_len_per_layer = [4096 * worker.block_size] - worker.num_blocks = 1 - worker.dst_num_blocks[worker.engine_id] = worker.num_blocks - - # Remote P with TP=8 also has 1 head/rank -> identical - # block_len despite tp_ratio == 2. - meta = NixlAgentMetadata( - engine_id=FakeNixlConnectorWorker.REMOTE_ENGINE_ID, - agent_metadata=FakeNixlWrapper.AGENT_METADATA, - kv_caches_base_addr=[0], - device_id=0, - num_blocks=1, - block_lens=list(worker.block_len_per_layer), - block_strides=list(worker.block_len_per_layer), - kv_cache_layout="LBHNC", - block_size=worker.block_size, - ssm_sizes=(0, 0), - attn_backend_name=worker.backend_name, - physical_blocks_per_logical_kv_block=1, - ) - - # Must validate cleanly (used to raise AssertionError). - worker.add_remote_agent(meta, remote_tp_size=8) - - @patch( - "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper", - FakeNixlWrapper, - ) - def test_handshake_rejects_wrong_block_len_without_gqa_replication( - self, default_vllm_config, dist_init - ): - """Ensure the head-ratio validation still rejects genuinely wrong - block_lens when GQA replication is NOT in effect (32 KV heads, - D_TP=4, P_TP=2: head_ratio=4, both sides have >1 head/rank). - """ - block_size = maybe_update_block_size(16) - vllm_config = create_vllm_config(block_size=block_size) - - with patch( - "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.get_tensor_model_parallel_world_size", # noqa: E501 - return_value=4, - ): - connector = NixlConnector( - vllm_config, - KVConnectorRole.WORKER, - make_kv_cache_config(block_size=block_size), - ) - connector.connector_worker = FakeNixlConnectorWorker( - vllm_config, connector.engine_id, hand_shake_latency=0 - ) - worker = connector.connector_worker - - worker.transfer_topo.total_num_kv_heads = 32 - worker.transfer_topo.local_physical_heads = 8 # 32 // 4 - worker.kv_cache_layout = "LBHNC" - - slot_size = 4096 - worker.slot_size_per_layer = [slot_size] - worker.block_len_per_layer = [slot_size * worker.block_size] - worker.num_blocks = 1 - worker.dst_num_blocks[worker.engine_id] = worker.num_blocks - - # Remote P_TP=2 has 16 heads/rank -> head_ratio = 16/8 = 2. - # Correct remote block_len = local * 2. Send local * 1 - # (wrong) to verify rejection. - bad_meta = NixlAgentMetadata( - engine_id=FakeNixlConnectorWorker.REMOTE_ENGINE_ID, - agent_metadata=FakeNixlWrapper.AGENT_METADATA, - kv_caches_base_addr=[0], - device_id=0, - num_blocks=1, - block_lens=list(worker.block_len_per_layer), - block_strides=list(worker.block_len_per_layer), - kv_cache_layout="LBHNC", - block_size=worker.block_size, - ssm_sizes=(0, 0), - attn_backend_name=worker.backend_name, - physical_blocks_per_logical_kv_block=1, - ) - - with pytest.raises(AssertionError): - worker.add_remote_agent(bad_meta, remote_tp_size=2) - - @pytest.mark.parametrize( - ("kv_quant_mode", "local_tp", "remote_tp", "num_kv_heads", "rejected"), - [ - (KVQuantMode.NVFP4, 2, 1, 8, True), - (KVQuantMode.NVFP4, 1, 2, 8, True), - (KVQuantMode.NVFP4, 2, 2, 8, False), - # Both sides hold one replicated head per rank: no head split. - (KVQuantMode.NVFP4, 16, 8, 8, False), - (KVQuantMode.NONE, 2, 1, 8, False), - ], - ) - @patch( - "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper", - FakeNixlWrapper, - ) - def test_handshake_rejects_nvfp4_kv_head_split( - self, - default_vllm_config, - dist_init, - kv_quant_mode, - local_tp, - remote_tp, - num_kv_heads, - rejected, - ): - """NVFP4 pages keep all heads' data before all heads' scales, so - heterogeneous-TP head splitting would silently corrupt the KV. - """ - block_size = maybe_update_block_size(16) - vllm_config = create_vllm_config(block_size=block_size) - kv_cache_config = KVCacheConfig( - num_blocks=1, - kv_cache_tensors=[], - kv_cache_groups=[ - KVCacheGroupSpec( - ["layer0"], - FullAttentionSpec( - block_size=block_size, - num_kv_heads=max(1, num_kv_heads // local_tp), - head_size=16, - dtype=torch.uint8, - kv_quant_mode=kv_quant_mode, - ), - ) - ], - ) - with patch( - "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.get_tensor_model_parallel_world_size", # noqa: E501 - return_value=local_tp, - ): - worker = FakeNixlConnectorWorker( - vllm_config, - "engine", - hand_shake_latency=0, - kv_cache_config=kv_cache_config, - ) - worker.transfer_topo.total_num_kv_heads = num_kv_heads - worker.transfer_topo.local_physical_heads = max(1, num_kv_heads // local_tp) - - local_len = 4096 * worker.block_size - remote_len = ( - local_len - * max(1, num_kv_heads // remote_tp) - // worker.transfer_topo.local_physical_heads - ) - worker.block_len_per_layer = [local_len] - worker.num_blocks = 1 - worker.dst_num_blocks[worker.engine_id] = worker.num_blocks - meta = NixlAgentMetadata( - engine_id=FakeNixlConnectorWorker.REMOTE_ENGINE_ID, - agent_metadata=FakeNixlWrapper.AGENT_METADATA, - kv_caches_base_addr=[0], - device_id=0, - num_blocks=1, - block_lens=[remote_len], - block_strides=[remote_len], - kv_cache_layout=worker.kv_cache_layout, - block_size=worker.block_size, - ssm_sizes=(0, 0), - attn_backend_name=worker.backend_name, - physical_blocks_per_logical_kv_block=1, - ) - - if rejected: - with pytest.raises(RuntimeError, match="NVFP4"): - worker.add_remote_agent(meta, remote_tp_size=remote_tp) - else: - worker.add_remote_agent(meta, remote_tp_size=remote_tp) - - -# NOTE: resource cleanup in mp backend is a bit finicky, so the order in which -# we put here is important. First run ray, it will clean up the resources, then -# the rest of the tests. -@patch( - "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper", - FakeNixlWrapper, -) -def test_kv_connector_stats(default_vllm_config, dist_init): - """Test that KV transfer stats are properly recorded and retrieved.""" - block_size = maybe_update_block_size(16) - vllm_config = create_vllm_config(block_size=block_size) - - # Test worker role in decode server. - connector = NixlConnector( - vllm_config, - KVConnectorRole.WORKER, - make_kv_cache_config(block_size=block_size), - ) - connector.connector_worker = FakeNixlConnectorWorker( - vllm_config, connector.engine_id, hand_shake_latency=0 - ) - - # Verify that xfer_stats starts empty - initial_stats = connector.get_kv_connector_stats() - assert initial_stats is None - - # Create transfer metadata - request_id = "test_req_for_stats" - metadata = NixlConnectorMetadata() - metadata.add_new_req_to_recv( - request_id=request_id, - local_block_ids=([0],), - kv_transfer_params={ - "remote_block_ids": ([0],), - "remote_engine_id": FakeNixlConnectorWorker.REMOTE_ENGINE_ID, - "remote_request_id": f"prefill-{request_id}", - "remote_host": "localhost", - "remote_port": 1234, - "remote_tp_size": 1, - }, - ) - connector.bind_connector_metadata(metadata) - - # Start the transfer - dummy_ctx = ForwardContext( - no_compile_layers={}, - attn_metadata={}, - slot_mapping={}, - ) - connector.start_load_kv(dummy_ctx) - - # Verify stats are recorded after transfer is complete - max_iterations = 2 - # Clear metadata before start_load_kv to prevent reprocessing same request - connector.bind_connector_metadata(NixlConnectorMetadata()) - for _ in range(max_iterations): - # Need to call start_load_kv to process completed handshakes - connector.start_load_kv(dummy_ctx) - _, done_recving = connector.get_finished(finished_req_ids=set()) - if len(done_recving) > 0 and request_id in done_recving: - break - time.sleep(0.1) # Small delay to allow background handshake to complete - else: - assert "Transfer did not complete within expected iterations" - - # Now check that stats were recorded - stats_after_transfer = connector.get_kv_connector_stats() - assert isinstance(stats_after_transfer, NixlKVConnectorStats) - - # Verify stats values are recorded - assert not stats_after_transfer.is_empty() - assert stats_after_transfer.num_successful_transfers == 1 - - # Verify stats are reset after retrieval - stats_after_reset = connector.get_kv_connector_stats() - assert stats_after_reset is None - - -@patch( - "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper", - FakeNixlWrapper, -) -def test_reqs_to_send_deadline_rebased_to_worker_clock(default_vllm_config, dist_init): - """reqs_to_send deadlines are stamped with the scheduler process's - perf_counter, whose epoch differs across processes and (by boot-time - deltas) across nodes. Without rebasing, a P worker on a node whose - monotonic clock is ahead of the scheduler's by more than the TTL - expires the lease on arrival and reports done_sending before D has - read the blocks — the freed blocks can then be reallocated and the - remote read pulls another request's data (silent accuracy corruption). - The worker must anchor the remaining TTL to its own clock. - """ - block_size = maybe_update_block_size(16) - vllm_config = create_vllm_config(block_size=block_size) - connector = NixlConnector( - vllm_config, - KVConnectorRole.WORKER, - make_kv_cache_config(block_size=block_size), - ) - connector.connector_worker = FakeNixlConnectorWorker( - vllm_config, connector.engine_id, hand_shake_latency=0 - ) - worker = connector.connector_worker - - req_id = "req-lease-clock" - ttl = 480.0 - # Simulate a scheduler whose monotonic clock is 10,000 s behind this - # worker's (e.g. its node booted much later): the raw deadline is - # then already far in the past in this worker's clock domain. - scheduler_clock = time.perf_counter() - 10_000.0 - - metadata = NixlConnectorMetadata() - metadata.reqs_in_batch = {req_id} - metadata.reqs_to_send = {req_id: scheduler_clock + ttl} - metadata.scheduler_clock = scheduler_clock - connector.bind_connector_metadata(metadata) - dummy_ctx = ForwardContext( - no_compile_layers={}, - attn_metadata={}, - slot_mapping={}, - ) - connector.start_load_kv(dummy_ctx) - - remaining = worker._reqs_to_send[req_id] - time.perf_counter() - assert ttl - 5.0 < remaining <= ttl + 5.0 - - # The expiry sweep must not release the request. - done_sending, _ = worker.get_finished() - assert req_id not in done_sending - assert req_id in worker._reqs_to_process - - -def test_kv_connector_stats_aggregation(): - """Test KV transfer stats aggregation across TP ranks using - KVOutputAggregator (used by MultiprocExecutor). - """ - # Create KVOutputAggregator for 3 workers (simulating TP=3), same thing - # done in MultiprocExecutor.execute_model - aggregator = KVOutputAggregator(expected_finished_count=3) - - # Create stats for multiple workers with different transfer patterns - worker1_stats = NixlKVConnectorStats() - worker2_stats = NixlKVConnectorStats() - worker3_stats = NixlKVConnectorStats() - - # Record different transfers on each worker - # Worker 1: 2 transfers - stats = get_default_xfer_telemetry() - worker1_stats.record_transfer(stats) - worker1_stats.record_transfer(stats) - - # Worker 2: 1 transfer - worker2_stats.record_transfer(stats) - - # Worker 3: 3 transfers - stats = get_default_xfer_telemetry( - xferDurationS=2, postDurationS=2, totalBytes=2, descCount=2 - ) - worker3_stats.record_transfer(stats) - worker3_stats.record_transfer(stats) - worker3_stats.record_transfer(stats) - - # Create ModelRunnerOutput instances for each worker - worker_outputs = [] - for i, worker_stats in enumerate([worker1_stats, worker2_stats, worker3_stats]): - output = ModelRunnerOutput( - req_ids=[f"req_{i}"], - req_id_to_index={f"req_{i}": 0}, - sampled_token_ids=[[123]], # dummy token - logprobs=None, - prompt_logprobs_dict={}, - pooler_output=[None], - kv_connector_output=KVConnectorOutput( - finished_sending=set([f"req_{i}_send"]) - if i < 2 - else None, # Workers 0,1 finished sending - finished_recving=set([f"req_{i}_recv"]) - if i > 0 - else None, # Workers 1,2 finished receiving - kv_connector_stats=worker_stats, - ), - ) - worker_outputs.append(output) - - # Use the real aggregation mechanism (like MultiprocExecutor.execute_model) - aggregated_output = aggregator.aggregate(worker_outputs, output_rank=0) - kv_connector_stats = aggregated_output.kv_connector_output.kv_connector_stats - assert isinstance(kv_connector_stats, NixlKVConnectorStats) - # Number of total transfers across all workers. - assert kv_connector_stats.num_successful_transfers == 6 - # Logging proc, call reduce() to get CLI-friendly stats. - cli_stats = kv_connector_stats.reduce() - assert cli_stats["Avg xfer time (ms)"] == 1500.0 - assert cli_stats["Avg post time (ms)"] == 1500.0 - assert cli_stats["Avg number of descriptors"] == 1.5 - # Reduced values must be plain Python scalars so CLI logging renders - # them without numpy reprs (eg np.float64(...)). - assert all(not isinstance(v, np.generic) for v in cli_stats.values()) - - -def test_kv_connector_stats_failure_grouping(): - """Transfer, handshake and notification failures are reported as one - transport-failure count, while KV expiry is reported separately: the - former are sporadic lower-transport-layer events, the latter an - autoscaler signal.""" - stats = NixlKVConnectorStats() - assert stats.is_empty() - - stats.record_failed_transfer() - stats.record_failed_handshake() - stats.record_failed_notification() - stats.record_kv_expired_req() - stats.record_notification_after_expiry() - assert not stats.is_empty() - - # No successful transfers: latency stats are zero but the failure - # counts still surface. - reduced = stats.reduce() - assert reduced["Num successful transfers"] == 0 - assert reduced["Num failed transfers"] == 3 - assert reduced["Num KV expired reqs"] == 1 - assert reduced["Num notifs after expiry"] == 1 - - -def test_nixl_prom_metrics_group_handshake_with_transfer_failures(): - """vllm:nixl_num_failed_transfers counts handshake and notification - failures too, while vllm:nixl_num_kv_expired_reqs stays a separate - counter.""" - from prometheus_client import CollectorRegistry, Counter, Gauge, Histogram - - from vllm.distributed.kv_transfer.kv_connector.v1.nixl.stats import ( - NixlPromMetrics, - ) - - registry = CollectorRegistry() - - class RegistryGauge(Gauge): - def __init__(self, *args, **kwargs): - super().__init__(*args, registry=registry, **kwargs) - - class RegistryCounter(Counter): - def __init__(self, *args, **kwargs): - super().__init__(*args, registry=registry, **kwargs) - - class RegistryHistogram(Histogram): - def __init__(self, *args, **kwargs): - super().__init__(*args, registry=registry, **kwargs) - - vllm_config = create_vllm_config() - metric_types = { - Gauge: RegistryGauge, - Counter: RegistryCounter, - Histogram: RegistryHistogram, - } - prom = NixlPromMetrics( - vllm_config, - metric_types, - labelnames=["engine"], - per_engine_labelvalues={0: ["engine-0"]}, - ) - - stats = NixlKVConnectorStats() - stats.record_failed_transfer() - stats.record_failed_handshake() - stats.record_failed_notification() - stats.record_kv_expired_req() - stats.record_notification_after_expiry() - prom.observe(stats.data, engine_idx=0) - - def counter_value(name: str) -> float: - for metric in registry.collect(): - for sample in metric.samples: - if sample.name == name: - return sample.value - raise AssertionError(f"metric {name} not found in registry") - - assert counter_value("vllm:nixl_num_failed_transfers_total") == 3.0 - assert counter_value("vllm:nixl_num_kv_expired_reqs_total") == 1.0 - assert counter_value("vllm:nixl_num_notifications_after_expiry_total") == 1.0 - - -@patch( - "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper", - FakeNixlWrapper, -) -def test_notification_after_expiry_is_counted(default_vllm_config, dist_init): - block_size = maybe_update_block_size(16) - vllm_config = create_vllm_config(block_size=block_size) - connector = NixlConnector( - vllm_config, - KVConnectorRole.WORKER, - make_kv_cache_config(block_size=block_size), - ) - connector.connector_worker = FakeNixlConnectorWorker( - vllm_config, connector.engine_id, hand_shake_latency=0 - ) - worker = connector.connector_worker - worker._reqs_to_process.add("known") - worker._reqs_to_send["known"] = time.perf_counter() + 10 - worker.nixl_wrapper.get_new_notifs = MagicMock( - return_value={"decode-agent": [b"known:1", b"unknown:1"]} - ) - - assert worker._get_new_notifs() == {"known"} - - stats = connector.get_kv_connector_stats() - assert isinstance(stats, NixlKVConnectorStats) - assert stats.data["num_notifications_after_expiry"] == [1] - - -def test_multi_kv_connector_stats_aggregation(): - """Test MultiKVConnectorStats aggregation across TP ranks using - KVOutputAggregator (used by MultiprocExecutor). - """ - aggregator = KVOutputAggregator(expected_finished_count=3) - - from dataclasses import dataclass - - # Mock a KVConnectorStats class for testing aggregation over connectors. - @dataclass - class FooKVConnectorStats(KVConnectorStats): - def reset(self): - self.data = {"num_foo_transfers": 0} - - def record_transfer(self): - if "num_foo_transfers" not in self.data: - self.data["num_foo_transfers"] = 0 - self.data["num_foo_transfers"] += 1 - - def is_empty(self) -> bool: - return self.data["num_foo_transfers"] == 0 - - def aggregate(self, other: "FooKVConnectorStats") -> "FooKVConnectorStats": - if not other.is_empty(): - self.data["num_foo_transfers"] += other.data["num_foo_transfers"] - return self - - def make_multi_stats(nixl_count: int, foo_count: int) -> MultiKVConnectorStats: - data: dict[str, KVConnectorStats] = {} - if nixl_count > 0: - nixl_stats = NixlKVConnectorStats() - for _ in range(nixl_count): - nixl_stats.record_transfer(get_default_xfer_telemetry()) - data["NixlConnector"] = nixl_stats - if foo_count > 0: - foo_stats = FooKVConnectorStats() - for _ in range(foo_count): - foo_stats.record_transfer() - data["FooConnector"] = foo_stats - return MultiKVConnectorStats(data=data) - - # Create heterogeneous stats across 3 workers - worker_patterns = [(2, 1), (3, 0), (0, 5)] # (Nixl, Foo) - - worker_outputs: list[ModelRunnerOutput] = [] - for i, (nixl_count, foo) in enumerate(worker_patterns): - stats = make_multi_stats(nixl_count, foo) - output = ModelRunnerOutput( - req_ids=[f"req_{i}"], - req_id_to_index={f"req_{i}": 0}, - sampled_token_ids=[[123]], - logprobs=None, - prompt_logprobs_dict={}, - pooler_output=[None], - kv_connector_output=KVConnectorOutput( - finished_sending=set([f"req_{i}_send"]) if i < 2 else None, - finished_recving=set([f"req_{i}_recv"]) if i > 0 else None, - kv_connector_stats=stats, - ), - ) - worker_outputs.append(output) - - aggregated_output = aggregator.aggregate(worker_outputs, output_rank=0) - kv_connector_stats = aggregated_output.kv_connector_output.kv_connector_stats - assert isinstance(kv_connector_stats, MultiKVConnectorStats) - - # Validate per-connector totals across workers - assert isinstance(kv_connector_stats["NixlConnector"], NixlKVConnectorStats) - assert kv_connector_stats["NixlConnector"].num_successful_transfers == 5 - assert isinstance(kv_connector_stats["FooConnector"], FooKVConnectorStats) - assert kv_connector_stats["FooConnector"].data["num_foo_transfers"] == 6 - - -@patch( - "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper", - FakeNixlWrapper, -) -def test_scheduler_kv_connector_stats_aggregation(): - """Test scheduler and worker KV connector stats aggregation.""" - from vllm.v1.core.sched.output import SchedulerOutput - - scheduler = create_scheduler(create_vllm_config()) - - # Worker stats with transfer metrics - worker_stats = NixlKVConnectorStats() - worker_stats.record_transfer(get_default_xfer_telemetry()) - - # Scheduler stats with custom metric (needs dummy transfer to avoid being skipped) - scheduler_stats = NixlKVConnectorStats() - scheduler_stats.data.update( - { # dummy transfer just for testing, to bypass is_empty() check - "transfer_duration": [0], - "post_duration": [0], - "bytes_transferred": [0], - "num_descriptors": [0], - } - ) - - # Mock the scheduler connector's stats method - scheduler.connector.get_kv_connector_stats = lambda: MultiKVConnectorStats( - data={"NixlConnector": scheduler_stats} - ) - - model_output = ModelRunnerOutput( - req_ids=["req_0"], - req_id_to_index={"req_0": 0}, - sampled_token_ids=[[123]], - logprobs=None, - prompt_logprobs_dict={}, - pooler_output=[None], - kv_connector_output=KVConnectorOutput( - kv_connector_stats=MultiKVConnectorStats( - data={"NixlConnector": worker_stats} - ) - ), - ) - scheduler_output = SchedulerOutput( - scheduled_new_reqs=[], - scheduled_cached_reqs=None, - num_scheduled_tokens={"req_0": 1}, - total_num_scheduled_tokens=1, - scheduled_spec_decode_tokens={}, - scheduled_encoder_inputs={}, - finished_req_ids=set(), - free_encoder_mm_hashes=[], - ) - - engine_core_outputs = scheduler.update_from_output(scheduler_output, model_output) - - final_stats = next( - iter(engine_core_outputs.values()) - ).scheduler_stats.kv_connector_stats - # The scheduler stats payload carries serialized per-connector dicts. - nixl_stats = final_stats["NixlConnector"] - assert len(nixl_stats["transfer_duration"]) == 2 - - -@pytest.mark.parametrize("distributed_executor_backend", ["ray", None]) -@patch( - "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper", - FakeNixlWrapper, -) -def test_abort_timeout_on_prefiller(monkeypatch, distributed_executor_backend): - """Test lifecycle of an aborted Remote Prefill request hitting the timeout. - -----> P - | {process request} - <-/--- | {result is NOT delivered, eg proxy is down} - | - | - | {eventually free blocks} - """ - model_name = "Qwen/Qwen3-0.6B" - timeout = 6 - kv_transfer_config = KVTransferConfig( - kv_connector="NixlConnector", - kv_role="kv_consumer", - kv_connector_extra_config={"kv_lease_duration": timeout}, - ) - llm_kwargs = { - "model": model_name, - "enforce_eager": True, - "gpu_memory_utilization": 0.5, - "kv_transfer_config": kv_transfer_config, - "distributed_executor_backend": distributed_executor_backend, - } - - monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0") - - def run_test_and_cleanup(): - llm = LLM(**llm_kwargs) - try: - _run_abort_timeout_test(llm, timeout) - finally: - llm.llm_engine.engine_core.shutdown() - - # Build runtime_env only if we're using Ray - if distributed_executor_backend == "ray": - with _make_fake_nixl_pkg() as working_dir: - runtime_env = { - "working_dir": working_dir, # ship fake nixl package - "env_vars": { - "NIXL_TELEMETRY_ENABLE": "1", - }, - } - # On XPU/ROCm, vLLM expects Ray's device key to be "GPU". - # Explicitly reserving GPU resources here prevents false negatives - # when Ray cannot auto-detect accelerator resources in test envs. - ray_init_kwargs: dict[str, Any] = {"runtime_env": runtime_env} - if not current_platform.is_cuda(): - ray_init_kwargs["num_gpus"] = 1 - ray.init(**ray_init_kwargs) - try: - run_test_and_cleanup() - finally: - ray.shutdown() - else: - run_test_and_cleanup() - - -class RequestIdMapper: - """Helper class to map external request IDs to internal request IDs.""" - - def __init__(self, output_processor: OutputProcessor): - self.req_id_mapping: dict[str, str] = {} - self.original_add_request = output_processor.add_request - output_processor.add_request = self._add_request - - def _add_request(self, request: EngineCoreRequest, *args, **kwargs): - self.req_id_mapping[request.external_req_id] = request.request_id - return self.original_add_request(request, *args, **kwargs) - - def __call__(self, external_req_id: str) -> str: - return self.req_id_mapping[external_req_id] - - -def _run_abort_timeout_test(llm: LLM, timeout: int): - """Helper function to run the abort timeout test logic.""" - remote_prefill_opts = { - "do_remote_decode": True, - "do_remote_prefill": False, - "remote_engine_id": None, - "remote_block_ids": None, - "remote_host": None, - "remote_port": None, - } - # Simulate sidecar request - sampling_params = SamplingParams( - temperature=0.0, - max_tokens=1, - extra_args={"kv_transfer_params": remote_prefill_opts}, - ) - scheduler = llm.llm_engine.engine_core.engine_core.scheduler - req_to_blocks = scheduler.kv_cache_manager.coordinator.single_type_managers[ - 0 - ].req_to_blocks - - id_mapper = RequestIdMapper(llm.llm_engine.output_processor) - - def req_id(outputs: list[RequestOutput]) -> str: - assert len(outputs) == 1 - return id_mapper(outputs[0].request_id) - - padding = "Just making this request a little longer so that we're sure " - "we're not hitting the small-request lower bound beneath which we don't " - "actually trigger the whole kv transfer, but rather just recompute the " - "blocks on D." - req0_id = req_id( - llm.generate([f"What is the capital of Japan? {padding}"], sampling_params) - ) - - # Request finished but not freed - assert req0_id in scheduler.finished_req_ids and req0_id in req_to_blocks - # Some other request, 0 still not freed - req1_id = req_id( - llm.generate([f"What is the capital of Italy? {padding}"], sampling_params) - ) - assert req0_id in req_to_blocks - assert req1_id in scheduler.finished_req_ids and req1_id in req_to_blocks - - # Wait for timeout and trigger another scheduler loop - time.sleep(timeout) - _ = llm.generate([f"What is the capital of France? {padding}"], sampling_params) - # Request-0 times out and is cleared! - assert req0_id not in req_to_blocks - # Need to shutdown the background thread to release NIXL side channel port - llm.llm_engine.engine_core.shutdown() - - -def test_mixed_memory_local_descriptors_split_by_memory_type(): - worker = object.__new__(NixlConnectorWorker) - worker.transfer_topo = MagicMock() - worker.block_size = 16 - worker.engine_id = "local" - worker.tp_rank = 0 - worker.device_id = 3 - worker.kv_caches_base_addr = {"local": {0: [100, 200]}} - worker._has_mamba = False - worker._mixed_mem_types = True - worker._skip_dram_xfer = False - worker.region_mem_types = ["DRAM", "VRAM"] - worker.region_num_blocks = [2, 2] - worker._desc_is_dram_by_block_size = {} - worker._desc_pos_by_block_size = {} - worker._dram_src_handles_by_block_size = {} - worker.nixl_memory_type = "VRAM" - # One dense run of two 10-byte blocks per region: (addr, len, dev, stride, count). - worker._build_fa_local = MagicMock( # type: ignore[method-assign] - return_value=np.array([[100, 10, 3, 10, 2], [200, 10, 3, 10, 2]]) - ) - worker.nixl_wrapper = MagicMock() - worker.nixl_wrapper.prep_xfer_dlist.side_effect = [11, 22] - - handle, blocks = worker.register_local_xfer_handler(worker.block_size) - - assert handle == 22 - assert worker._dram_src_handles_by_block_size[worker.block_size] == 11 - # Must stay an array: P_TP > D_TP splits call .tolist() on it. - assert isinstance(blocks, np.ndarray) - # DRAM runs are re-homed to device 0; the per-descriptor maps consumed by - # the mixed READ path still cover every block. - assert blocks[:, 2].tolist() == [0, 3] - assert worker._desc_is_dram_by_block_size[16].tolist() == [True, True, False, False] - assert worker._desc_pos_by_block_size[16].tolist() == [0, 1, 0, 1] - dlist_calls = worker.nixl_wrapper.prep_xfer_dlist.call_args_list - assert [call.kwargs["mem_type"] for call in dlist_calls] == ["DRAM", "VRAM"] - assert [call.args[1].tolist() for call in dlist_calls] == [ - [[100, 10, 0, 10, 2]], - [[200, 10, 3, 10, 2]], - ] - - -@pytest.fixture -def recv_worker(): - """Receive lifecycle state without distributed or device initialization.""" - worker = object.__new__(NixlConnectorWorker) - worker.transfer_topo = MagicMock() - worker.transfer_topo.block_size_ratio.return_value = 1 - worker.transfer_topo.get_engine_info.return_value = SimpleNamespace( - remote_block_size=16, remote_physical_blocks_per_logical=1 - ) - worker._physical_blocks_per_logical_kv_block = 1 - worker._recving_metadata = {"request": MagicMock(local_block_ids=([1, 2, 3],))} - worker._recving_transfers = defaultdict(list) - worker._failed_recv_reqs = queue.Queue() - worker._recv_failures = set() - worker._handshake_lock = threading.RLock() - worker._handshake_futures = {} - worker._remote_agents = {} - worker._engine_by_address = {} - worker._failed_remote_engines = set() - worker._invalid_remote_engines = set() - worker._replicated_pcp_done_sending = set() - worker._invalid_block_ids = queue.Queue() - worker._pending_recv_notifs = {} - worker._reqs_to_send = {} - worker._replicated_pcp_done_sending = set() - worker._is_hma_required = False - worker._has_mamba = False - worker.use_host_buffer = False - worker.enable_permute_local_kv = False - worker.enable_heterogeneous_attn_post_process = False - worker.tp_rank = 0 - worker._skip_dram_xfer = False - worker._log_failure = MagicMock() # type: ignore[method-assign] - worker.xfer_stats = NixlKVConnectorStats() - worker.nixl_wrapper = MagicMock() - worker.nixl_wrapper.get_xfer_telemetry.return_value = get_default_xfer_telemetry() - return worker - - -def test_mixed_memory_read_notifies_after_both_transfers_finish(recv_worker): - worker = recv_worker - worker._desc_is_dram_by_block_size = {16: np.array([True, True, False, False])} - worker._desc_pos_by_block_size = {16: np.array([0, 1, 0, 1])} - worker._dram_src_handles_by_block_size = {16: 10} - worker.nixl_wrapper.make_prepped_xfer.side_effect = [101, 102] - worker.nixl_wrapper.check_xfer_state.return_value = "DONE" - - worker._read_blocks_mixed( - request_id="request", - local_block_size_key=16, - local_device_handle=20, - local_dram_handle=10, - remote_xfer_side_handle=30, - local_block_descs_ids=np.array([0, 2]), - remote_block_descs_ids=np.array([5, 7]), - notif_agent="prefill", - notif_id=b"request:1", - ) - - dram_read, device_read = worker.nixl_wrapper.make_prepped_xfer.call_args_list - assert dram_read.args[:2] == ("READ", 10) - assert dram_read.args[3] == 30 - np.testing.assert_array_equal(dram_read.args[2], [0]) - np.testing.assert_array_equal(dram_read.args[4], [5]) - assert device_read.args[:2] == ("READ", 20) - assert device_read.args[3] == 30 - np.testing.assert_array_equal(device_read.args[2], [0]) - np.testing.assert_array_equal(device_read.args[4], [7]) - worker.nixl_wrapper.send_notif.assert_not_called() - - assert worker.get_finished() == (set(), {"request"}) - worker.nixl_wrapper.send_notif.assert_called_once_with( - "prefill", notif_msg=b"request:1" - ) - - -def test_mixed_memory_read_failure_does_not_notify_producer(recv_worker): - """A failed half of a split READ must suppress its success notification.""" - worker = recv_worker - worker._recving_transfers = {"request": [101, 102]} - worker._pending_recv_notifs = {"request": [("prefill", b"request:1")]} - worker.nixl_wrapper.check_xfer_state.side_effect = ["ERR", "DONE"] - - assert worker.get_finished() == (set(), {"request"}) - - assert "request" not in worker._pending_recv_notifs - worker.nixl_wrapper.send_notif.assert_not_called() - - -def test_hisparse_dram_rank0_only_read_keeps_device_part(recv_worker): - worker = recv_worker - worker.tp_rank = 1 - worker._skip_dram_xfer = True - worker._desc_is_dram_by_block_size = {16: np.array([True, True, False, False])} - worker._desc_pos_by_block_size = {16: np.array([0, 1, 0, 1])} - worker._dram_src_handles_by_block_size = {16: None} - worker.nixl_wrapper.make_prepped_xfer.side_effect = [102] - worker.nixl_wrapper.check_xfer_state.return_value = "DONE" - - worker._read_blocks_mixed( - request_id="request", - local_block_size_key=16, - local_device_handle=20, - local_dram_handle=None, - remote_xfer_side_handle=30, - local_block_descs_ids=np.array([0, 2]), - remote_block_descs_ids=np.array([5, 7]), - notif_agent="prefill", - notif_id=b"request:1", - ) - - (device_read,) = worker.nixl_wrapper.make_prepped_xfer.call_args_list - assert device_read.args[:2] == ("READ", 20) - np.testing.assert_array_equal(device_read.args[2], [0]) - np.testing.assert_array_equal(device_read.args[4], [7]) - - assert worker.get_finished() == (set(), {"request"}) - worker.nixl_wrapper.send_notif.assert_called_once_with( - "prefill", notif_msg=b"request:1" - ) - - -def test_hisparse_dram_rank0_only_dram_only_read_completes(recv_worker): - """With nothing left to read, the request still completes and notifies.""" - worker = recv_worker - worker.tp_rank = 1 - worker._skip_dram_xfer = True - worker._desc_is_dram_by_block_size = {16: np.array([True, True, False, False])} - worker._desc_pos_by_block_size = {16: np.array([0, 1, 0, 1])} - - worker._read_blocks_mixed( - request_id="request", - local_block_size_key=16, - local_device_handle=20, - local_dram_handle=None, - remote_xfer_side_handle=30, - local_block_descs_ids=np.array([0, 1]), - remote_block_descs_ids=np.array([5, 6]), - notif_agent="prefill", - notif_id=b"request:1", - ) - - worker.nixl_wrapper.make_prepped_xfer.assert_not_called() - assert worker.get_finished() == (set(), {"request"}) - worker.nixl_wrapper.send_notif.assert_called_once_with( - "prefill", notif_msg=b"request:1" - ) - - -def element_byte_addrs(view: torch.Tensor) -> list[int]: - """Absolute byte addresses of every element of a (possibly strided) view.""" - offsets = torch.zeros(view.shape, dtype=torch.int64) - for dim, (size, stride) in enumerate(zip(view.shape, view.stride())): - shape = [1] * view.ndim - shape[dim] = size - offsets = offsets + torch.arange(size, dtype=torch.int64).view(shape) * stride - esize = view.element_size() - byte_offsets = (offsets.flatten() * esize).tolist() - base = view.data_ptr() - return [base + offset + i for offset in byte_offsets for i in range(esize)] - - -@pytest.mark.parametrize( - "attn_backend", - [ - pytest.param( - "FLASH_ATTN", - marks=pytest.mark.skipif( - current_platform.is_rocm(), - reason="Attention backend FLASH_ATTN is not supported on ROCm", - ), - ), - "TRITON_ATTN", - ], -) -@pytest.mark.parametrize("layout", [layout.name for layout in KVCacheLayout]) -@pytest.mark.parametrize("separate_kv_head_groups", [False, True]) -def test_register_kv_caches( - default_vllm_config, - dist_init, - attn_backend, - layout, - separate_kv_head_groups, -): - """Test that register_kv_caches() properly calls nixl_wrapper methods with - correct data. - - This test verifies: - 1. nixl_wrapper.get_reg_descs() is called with caches_data containing - tensor metadata - 2. nixl_wrapper.prep_xfer_dlist() is called with strided block runs whose - expansion covers exactly the registered blocks - """ - block_size = maybe_update_block_size(16) - vllm_config = create_vllm_config( - attention_backend=attn_backend, block_size=block_size - ) - vllm_config.cache_config.kv_cache_layout = layout - - # Import the appropriate backend based on the parameter - if attn_backend == "FLASH_ATTN": - from vllm.v1.attention.backends.flash_attn import FlashAttentionBackend - - backend_cls = FlashAttentionBackend - elif attn_backend == "ROCM_ATTN": - from vllm.v1.attention.backends.rocm_attn import RocmAttentionBackend - - backend_cls = RocmAttentionBackend - else: # TRITON_ATTN - from vllm.v1.attention.backends.triton_attn import TritonAttentionBackend - - backend_cls = TritonAttentionBackend - - nixl_worker = "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker" - with ( - patch(f"{nixl_worker}.NixlWrapper") as mock_nixl_wrapper, - patch(f"{nixl_worker}.threading.Event"), - patch(f"{nixl_worker}.threading.Thread") as mock_thread, - patch(f"{nixl_worker}.get_current_attn_backends") as mock_get_attn_backends, - ): - mock_get_attn_backends.return_value = [backend_cls] - num_blocks = 8 - num_heads = 4 - head_size = 16 - - kv_cache_spec = FullAttentionSpec( - block_size=block_size, - num_kv_heads=num_heads, - head_size=head_size, - dtype=torch.float16, - num_head_slots=2 if separate_kv_head_groups else None, - state_content_bytes=num_heads * head_size * 2 - if separate_kv_head_groups - else None, - ) - kv_cache_config = KVCacheConfig( - num_blocks=num_blocks, - kv_cache_tensors=[], - kv_cache_groups=[ - KVCacheGroupSpec( - ["layer0", "layer1", "layer2", "layer3"], kv_cache_spec - ) - ], - ) - # Create connector - connector = NixlConnector(vllm_config, KVConnectorRole.WORKER, kv_cache_config) - connector.connector_worker = FakeNixlConnectorWorker( - vllm_config, - connector.engine_id, - hand_shake_latency=0, - kv_cache_config=kv_cache_config, - ) - - # Get the mock instance - mock_wrapper_instance = mock_nixl_wrapper.return_value - connector.connector_worker.nixl_wrapper = mock_wrapper_instance - - # Appease NixlHandshakePayload encoding with some bytes - mock_wrapper_instance.get_agent_metadata.return_value = b"fake_agent_metadata" - - # Reassure the shutdown() check that the thread is terminated - mock_thread.return_value.is_alive.return_value = False - - raw0 = torch.zeros( - kv_cache_spec.page_size_bytes * kv_cache_config.num_blocks * 2, - dtype=torch.int8, - device=current_platform.device_type, - ) - raw1 = torch.zeros( - kv_cache_spec.page_size_bytes * kv_cache_config.num_blocks, - dtype=torch.int8, - device=current_platform.device_type, - ) - tensor0, tensor1 = dense_kv_cache_views( - raw0, - kv_cache_spec, - kv_cache_config.num_blocks, - num_layers=2, - layout=KVCacheLayout[layout], - ) - (tensor2,) = dense_kv_cache_views( - raw1, - kv_cache_spec, - kv_cache_config.num_blocks, - num_layers=1, - layout=KVCacheLayout[layout], - ) - kv_caches = { - "layer0": tensor0, - "layer1": tensor1, - "layer2": tensor2, - "layer3": tensor0, - } - - # Execute register_kv_caches - connector.register_kv_caches(kv_caches) - - # Verify get_reg_descs was called with caches_data - assert mock_wrapper_instance.get_reg_descs.called - caches_data, _ = mock_wrapper_instance.get_reg_descs.call_args[0] - assert len(caches_data) == 2 - - for cache_entry, raw in zip(caches_data, (raw0, raw1)): - base_addr, size, _tp_rank, _ = cache_entry - assert size == raw.nbytes - assert base_addr == raw.data_ptr() - - # Verify the local dlist was prepared from one strided run per region; - # expand it to per-block windows for the layout-blind checks below. - assert mock_wrapper_instance.prep_xfer_dlist.called - _agent, block_runs = mock_wrapper_instance.prep_xfer_dlist.call_args[0] - assert block_runs.shape[1] == 5 - blocks_data = expand_strided_descs(block_runs) - - # Layout-blind contract: whatever regions the worker carves out, - # transferring "block b" must move exactly logical block b's bytes for - # every layer. Map each registered byte to the block that owns it, then - # require every descriptor window to hold bytes of a single block and - # the windows of block b to cover exactly block b's bytes. - owner: dict[int, int] = {} - for cache in (tensor0, tensor1, tensor2): - for blk in range(num_blocks): - for addr in element_byte_addrs(cache[blk]): - assert owner.setdefault(addr, blk) == blk - block_bytes = defaultdict(set) - for addr, blk in owner.items(): - block_bytes[blk].add(addr) - - covered: defaultdict[int, set[int]] = defaultdict(set) - for block_start_addr, block_len, _tp_rank in blocks_data: - window = range(block_start_addr, block_start_addr + block_len) - owners = {owner[addr] for addr in window} - assert len(owners) == 1, "descriptor window spans logical blocks" - covered[owners.pop()].update(window) - assert covered == block_bytes - - # Region bases are exactly the block-0 window starts, one per region. - base_addrs = connector.connector_worker.kv_caches_base_addr[ - connector.connector_worker.engine_id - ][0] - if layout == "BLHNC": - assert len(base_addrs) == 3 - assert set(base_addrs) == { - start for start, _len, _tp in blocks_data if owner[start] == 0 - } - assert len(blocks_data) == num_blocks * len(base_addrs) - assert len(block_runs) == len(base_addrs) - - assert connector.connector_worker.block_size == block_size - - -def _dsv4_mla_spec(record_bytes: int, alignment: int, tokens_per_state: int): - return MLAAttentionSpec( - block_size=256, - num_kv_heads=1, - head_size=512, - dtype=torch.uint8, - cache_dtype_str="fp8_ds_mla", - tokens_per_state=tokens_per_state, - alignment=alignment, - model_version="deepseek_v4", - state_content_bytes=record_bytes, - ) - - -@pytest.mark.parametrize( - "groups", - [ - # V4 record: padded pages, so a layer view is not a contiguous page. - [(_dsv4_mla_spec(584, 576, 4), 4)], - # V4.1 MXFP8 record: unpadded pages in rows that the compressed caches - # and a compressor-state ring alias. - [ - (_dsv4_mla_spec(528, 512, 2), 2), - (_dsv4_mla_spec(528, 512, 2), 2), - ( - CircularBufferSpec( - block_size=8, - num_kv_heads=1, - head_size=8448, - head_size_v=0, - dtype=torch.uint8, - ), - 2, - ), - ], - ], - ids=["v4-padded-pages", "v41-aliased-groups"], -) -def test_register_packed_dsv4_mla_cache_as_single_region( - default_vllm_config, dist_init, groups -): - from vllm.v1.attention.backends.triton_attn import TritonAttentionBackend - - nixl_worker = "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker" - with ( - patch(f"{nixl_worker}.NixlWrapper") as mock_nixl_wrapper, - patch(f"{nixl_worker}.threading.Event"), - patch(f"{nixl_worker}.threading.Thread") as mock_thread, - patch(f"{nixl_worker}.get_current_attn_backends") as mock_backends, - ): - mock_backends.return_value = [TritonAttentionBackend] - num_blocks = 2 - specs = [spec for spec, _ in groups] - group_layers = [ - [f"group{group_idx}.layer{idx}" for idx in range(num_layers)] - for group_idx, (_, num_layers) in enumerate(groups) - ] - kv_cache_config = KVCacheConfig( - num_blocks=num_blocks, - kv_cache_tensors=[], - kv_cache_groups=[ - KVCacheGroupSpec(layer_names, spec) - for layer_names, spec in zip(group_layers, specs) - ], - ) - vllm_config = create_vllm_config(attention_backend="TRITON_ATTN") - vllm_config.cache_config.block_size = 256 - vllm_config.cache_config.kv_cache_layout = "BLHNC" - connector = NixlConnector(vllm_config, KVConnectorRole.WORKER, kv_cache_config) - connector.connector_worker = FakeNixlConnectorWorker( - vllm_config, - connector.engine_id, - hand_shake_latency=0, - kv_cache_layout="BLHNC", - kv_cache_config=kv_cache_config, - ) - wrapper = mock_nixl_wrapper.return_value - connector.connector_worker.nixl_wrapper = wrapper - wrapper.get_agent_metadata.return_value = b"fake_agent_metadata" - mock_thread.return_value.is_alive.return_value = False - connector.connector_worker.use_mla = True - - # Every group packs its layers' pages into the same block rows. - (packed_block_len,) = { - len(layer_names) * spec.page_size_bytes - for layer_names, spec in zip(group_layers, specs) - } - raw = torch.zeros( - packed_block_len * num_blocks, - dtype=torch.int8, - device=current_platform.device_type, - ) - kv_caches: dict[str, torch.Tensor] = {} - for layer_names, spec in zip(group_layers, specs): - views = dense_kv_cache_views( - raw, spec, num_blocks, len(layer_names), KVCacheLayout.BLHNC - ) - kv_caches.update(zip(layer_names, views)) - connector.register_kv_caches(kv_caches) - - # MLA pages transfer as whole rows; a ring page past the row start keeps - # its own region. - ring_pages = [ - (cache.data_ptr(), spec.page_size_bytes) - for layer_names, spec in zip(group_layers, specs) - if isinstance(spec, CircularBufferSpec) - for cache in (kv_caches[name] for name in layer_names) - if cache.data_ptr() != raw.data_ptr() - ] - _agent, block_runs = wrapper.prep_xfer_dlist.call_args[0] - assert connector.connector_worker.kv_caches_base_addr[ - connector.connector_worker.engine_id - ][0] == [raw.data_ptr()] + [addr for addr, _ in ring_pages] - assert block_runs.tolist() == [ - [raw.data_ptr(), packed_block_len, 0, packed_block_len, num_blocks] - ] + [ - [addr, page_size, 0, packed_block_len, num_blocks] - for addr, page_size in ring_pages - ] - - -class FakePlatform(Platform): - device_type: str = "oot" - - @classmethod - def get_nixl_supported_devices(cls) -> dict[str, tuple[str, ...]]: - """Returns a mapping from device_type to a tuple of supported - kv_buffer_device for nixl. - """ - return {"oot": ("oot",)} - - @classmethod - def get_nixl_memory_type(cls) -> str | None: - """Returns the nixl memory type for the current platform.""" - return "VRAM" - - -@pytest.mark.parametrize( - "kv_buffer_device, nixl_memory_type", - [ - ("oot", "VRAM"), - ], -) -def test_kv_buffer_to_nixl_memory_types( - default_vllm_config, dist_init, kv_buffer_device, nixl_memory_type -): - """Test that register_kv_caches() passes the correct memory types from the - config to the nixl_wrapper. - """ - block_size = maybe_update_block_size(16) - vllm_config = create_vllm_config(block_size=block_size) - # Override the default memory types in the config - vllm_config.kv_transfer_config.kv_buffer_device = kv_buffer_device - from vllm.distributed.kv_transfer.kv_connector.v1.nixl.utils import ( - _NIXL_SUPPORTED_DEVICE, - ) - - _NIXL_SUPPORTED_DEVICE.update(FakePlatform.get_nixl_supported_devices()) - - with ( - patch( - "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper" - ), - patch( - "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.threading.Event" - ), - patch( - "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.threading.Thread" - ), - patch( - "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.current_platform", - FakePlatform, - ), - patch( - "vllm.distributed.kv_transfer.kv_connector.v1.nixl.utils._NIXL_SUPPORTED_DEVICE", - _NIXL_SUPPORTED_DEVICE, - ), - ): # noqa: E501 - # Create connector and replace its worker with a fake one for isolation - connector = NixlConnector( - vllm_config, - KVConnectorRole.WORKER, - make_kv_cache_config(block_size=block_size), - ) - - # Verify get_reg_descs was called with the correct memory_type - assert connector.connector_worker.kv_buffer_device == kv_buffer_device - assert connector.connector_worker.nixl_memory_type == nixl_memory_type - - -@patch( - "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper", - FakeNixlWrapper, -) -def test_shutdown_cleans_up_resources(default_vllm_config, dist_init): - """Test that shutdown() properly cleans up all resources.""" - block_size = maybe_update_block_size(16) - vllm_config = create_vllm_config(block_size=block_size) - - scheduler = NixlConnectorScheduler( - vllm_config, - vllm_config.kv_transfer_config.engine_id, - make_kv_cache_config(block_size=block_size), - ) - worker = NixlConnectorWorker( - vllm_config, - vllm_config.kv_transfer_config.engine_id, - make_kv_cache_config(block_size=block_size), - ) - nixl_wrapper = worker.nixl_wrapper - - with ( - patch.object(worker, "_handshake_initiation_executor") as mock_exec, - patch.object(scheduler, "_nixl_handshake_listener_t") as mock_listener, - patch.object(nixl_wrapper, "release_xfer_handle") as mock_rel_xfer, - patch.object(nixl_wrapper, "release_dlist_handle") as mock_rel_dlist, - patch.object(nixl_wrapper, "remove_remote_agent") as mock_rem_agent, - patch.object(nixl_wrapper, "deregister_memory") as mock_dereg, - ): - worker._recving_transfers = {"req1": [123]} - # Mock register_kv_cache which registers local handle - worker.src_xfer_handles_by_block_size = {worker.block_size: 455} - # P TP = 2 * D TP case, we should register 2 local handles - worker.src_xfer_handles_by_tp_ratio = {(-2, block_size): [456, 457]} - worker.dst_xfer_side_handles = {"engine1": {0: 789}} - worker._remote_agents = {"engine1": {(0, 0): "agent1"}} - # _cleanup_remote_engine (called by shutdown) also clears these: - worker.kv_caches_base_addr["engine1"] = {0: [0xABC]} - worker.dst_num_blocks["engine1"] = 50 - worker.tp_mappings["engine1"] = MagicMock() - worker._engine_last_active["engine1"] = time.perf_counter() - worker._registered_descs = ["desc1", "desc2"] - - mock_listener.is_alive.return_value = False - - worker.shutdown() - - # Test idempotency - worker.shutdown() - worker.shutdown() - - mock_exec.shutdown.assert_called_with(wait=False) - - # Same sequence on scheduler.shutdown() - scheduler.shutdown() - scheduler.shutdown() - scheduler.shutdown() - mock_listener.join.assert_called_once() - - mock_rel_xfer.assert_called_once_with(123) - assert mock_rel_dlist.call_count == 4 - mock_rel_dlist.assert_any_call(455) # src handle (whole region) - mock_rel_dlist.assert_any_call(456) # src handle (1st chunk) - mock_rel_dlist.assert_any_call(457) # src handle (2nd chunk) - mock_rel_dlist.assert_any_call(789) # dst handle - mock_rem_agent.assert_called_once_with("agent1") - assert mock_dereg.call_count == 2 - mock_dereg.assert_any_call("desc1") - mock_dereg.assert_any_call("desc2") - - -# ── TTL-based remote engine eviction tests ────────────────────────── - - -def _setup_worker_with_remote_engine( - engine_ttl: float = 10.0, -) -> tuple[Any, str]: - """Create a worker with one remote engine registered.""" - block_size = maybe_update_block_size(16) - vllm_config = create_vllm_config( - block_size=block_size, - kv_connector_extra_config={"engine_ttl": engine_ttl}, - ) - worker = NixlConnectorWorker( - vllm_config, - vllm_config.kv_transfer_config.engine_id, - make_kv_cache_config(block_size=block_size), - ) - - engine_id = "remote-engine-1" - worker._remote_agents[engine_id] = {(0, 0): "agent_0", (0, 1): "agent_1"} - worker.dst_xfer_side_handles[engine_id] = {0: 100, 1: 200} - worker.kv_caches_base_addr[engine_id] = {0: [0xABC]} - worker.dst_num_blocks[engine_id] = 50 - worker.tp_mappings[engine_id] = MagicMock() - worker._engine_last_active[engine_id] = time.perf_counter() - - worker.transfer_topo = MagicMock() - - return worker, engine_id - - -@patch( - "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper", - FakeNixlWrapper, -) -def test_engine_ttl_eviction(default_vllm_config, dist_init): - """Stale engines are evicted when TTL expires.""" - worker, engine_id = _setup_worker_with_remote_engine(engine_ttl=10.0) - nixl_wrapper = worker.nixl_wrapper - - with ( - patch.object(nixl_wrapper, "release_dlist_handle") as mock_rel, - patch.object(nixl_wrapper, "remove_remote_agent") as mock_rem, - ): - # Make the engine stale. - worker._engine_last_active[engine_id] = time.perf_counter() - 20.0 - - worker._evict_stale_engines() - - assert engine_id not in worker._remote_agents - assert engine_id not in worker.dst_xfer_side_handles - assert engine_id not in worker.kv_caches_base_addr - assert engine_id not in worker.dst_num_blocks - assert engine_id not in worker.tp_mappings - assert engine_id not in worker._engine_last_active - worker.transfer_topo.unregister_remote_engine.assert_called_with(engine_id) - - assert mock_rel.call_count == 2 - mock_rel.assert_any_call(100) - mock_rel.assert_any_call(200) - - assert mock_rem.call_count == 2 - mock_rem.assert_any_call("agent_0") - mock_rem.assert_any_call("agent_1") - - -@patch( - "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper", - FakeNixlWrapper, -) -def test_engine_with_inflight_transfer_is_not_evicted(default_vllm_config, dist_init): - """A transfer that outlives the TTL must keep its engine registered. - - _engine_last_active is stamped when a read is issued and not refreshed - while it runs, so a stalled transfer -- a peer that has lost its NIC holds - one indefinitely -- leaves its engine looking idle. Evicting it releases - the dlist handle and remote agent the transfer is still reading through. - """ - worker, engine_id = _setup_worker_with_remote_engine(engine_ttl=10.0) - nixl_wrapper = worker.nixl_wrapper - - request_id = "req-still-reading" - worker._recving_transfers[request_id] = [MagicMock()] - worker._recving_metadata[request_id] = MagicMock( - remote=MagicMock(engine_id=engine_id) - ) - - with ( - patch.object(nixl_wrapper, "release_dlist_handle") as mock_rel, - patch.object(nixl_wrapper, "remove_remote_agent") as mock_rem, - ): - worker._engine_last_active[engine_id] = time.perf_counter() - 20.0 - - worker._evict_stale_engines() - - assert engine_id in worker._remote_agents - assert engine_id in worker.dst_xfer_side_handles - mock_rel.assert_not_called() - mock_rem.assert_not_called() - - # Once the transfer is done the engine is stale like any other. - del worker._recving_transfers[request_id] - worker._evict_stale_engines() - - assert engine_id not in worker._remote_agents - assert mock_rem.call_count == 2 - - -@patch( - "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper", - FakeNixlWrapper, -) -def test_heartbeated_engine_is_not_evicted(default_vllm_config, dist_init, monkeypatch): - """Heartbeats keep a remote engine's NIXL state alive. - - Requests waiting in the D scheduler issue no reads, but their engine is - heartbeated through its remote agents. If the TTL expires anyway, the - next heartbeat evicts the engine and starts a new handshake, which loads - every remote agent again. - """ - from vllm.distributed.kv_transfer.kv_connector.v1.nixl import base_worker - - worker, engine_id = _setup_worker_with_remote_engine(engine_ttl=10.0) - nixl_wrapper = worker.nixl_wrapper - - now = {"t": 1000.0} - monkeypatch.setattr(base_worker.time, "perf_counter", lambda: now["t"]) - worker._engine_last_active[engine_id] = now["t"] - - metadata = NixlConnectorMetadata() - metadata.heartbeat_by_engine = { - engine_id: HeartbeatInfo( - req_ids={"remote-req"}, host="localhost", port=1234, tp_size=2 - ) - } - - with ( - patch.object(nixl_wrapper, "send_notif") as mock_send, - patch.object(nixl_wrapper, "remove_remote_agent") as mock_rem, - patch.object(worker, "_handshake_initiation_executor") as mock_executor, - ): - # One heartbeat every 5 s for twice the TTL, and no reads. - for _ in range(4): - now["t"] += 5.0 - worker._send_heartbeats(metadata) - - assert engine_id in worker._remote_agents - mock_rem.assert_not_called() - mock_executor.submit.assert_not_called() - # Every heartbeat reached both remote agents. - assert mock_send.call_count == 8 - - -@patch( - "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper", - FakeNixlWrapper, -) -def test_engine_ttl_disabled(default_vllm_config, dist_init): - """Eviction is disabled when engine_ttl <= 0.""" - worker, engine_id = _setup_worker_with_remote_engine(engine_ttl=0.0) - - # Make the engine stale. - worker._engine_last_active[engine_id] = time.perf_counter() - 9999.0 - - worker._evict_stale_engines() - - # Nothing should be evicted. - assert engine_id in worker._remote_agents - assert engine_id in worker.dst_xfer_side_handles - - -@pytest.mark.cpu_test -class TestPeerReplacement: - @pytest.fixture(autouse=True) - def setup(self, monkeypatch): - from vllm.distributed.kv_transfer.kv_connector.v1.nixl import base_worker as bw - from vllm.v1.attention.backends.flash_attn import FlashAttentionBackend - - block_size = maybe_update_block_size(16) - - config = create_vllm_config( - kv_connector_extra_config={"engine_ttl": 0}, block_size=block_size - ) - config.kv_transfer_config.kv_buffer_device = "cpu" - platform = SimpleNamespace( - device_type="cpu", - discover_numa_topology=lambda: [], - get_nixl_memory_type=lambda: "DRAM", - is_rocm=lambda: False, - ) - with ( - patch.object(bw, "NixlWrapper", FakeNixlWrapper), - patch.object(bw, "current_platform", platform), - patch.object(bw, "get_tensor_model_parallel_rank", return_value=0), - patch.object(bw, "get_tensor_model_parallel_world_size", return_value=1), - patch.object( - bw, "get_current_attn_backends", return_value=[FlashAttentionBackend] - ), - ): - self.worker = FakeNixlConnectorWorker(config, "local", hand_shake_latency=0) - self.nixl = self.worker.nixl_wrapper - self.transport = MagicMock(wraps=self.nixl) - self.worker.nixl_wrapper = self.transport - monkeypatch.setattr( - self.worker._handshake_initiation_executor, - "submit", - lambda *args: Future(), - ) - self._connect("old") - yield - self.worker.shutdown() - - def _connect(self, engine_id="new", host="localhost", port=1234): - future = self.worker._ensure_handshake(engine_id, host, port, 1) - self.worker.REMOTE_ENGINE_ID = engine_id - self.nixl.REMOTE_AGENT_NAME = engine_id - future.set_result(self.worker._nixl_handshake(host, port, 1, engine_id)) - - def _request(self, engine_id="new", blocks=(1,), awaiting_kvs=True, req_id=None): - metadata = NixlConnectorMetadata() - metadata.reqs_to_recv[req_id or f"{engine_id}-req"] = ReqMeta( - local_block_ids=(blocks,), - local_physical_block_ids=(blocks,), - tp_size=1, - remote=RemoteMeta( - ([2],), "localhost", 1234, engine_id, f"{engine_id}-prefill" - ), - awaiting_kvs=awaiting_kvs, - ) - return metadata - - def _fail_old_peer(self): - self.worker.start_load_kv(self._request("old")) - self.nixl.invalid_remote_agents.add("old") - self.transport.check_xfer_state.return_value = "ERR" - - def test_cleans_old_peer_after_confirmed_handshake(self): - self._connect("healthy", host="other-host") - old_handle = self.worker.dst_xfer_side_handles["old"][0] - self.worker.start_load_kv(self._request()) - self.worker.get_transfer_results() - self.transport.remove_remote_agent.assert_not_called() - - self._connect() - self.worker.get_transfer_results() - self.transport.release_dlist_handle.assert_called_once_with(old_handle) - self.transport.remove_remote_agent.assert_called_once_with("old") - assert set(self.worker._remote_agents) == {"new", "healthy"} - assert self.worker._engine_by_address == { - ("localhost", 1234): "new", - ("other-host", 1234): "healthy", - } - with pytest.raises(KeyError): - self.worker.transfer_topo.get_engine_info("old") - - self.worker.start_load_kv(NixlConnectorMetadata()) - assert self.worker.get_transfer_results().finished_recving == {"new-req"} - self.transport.remove_remote_agent.assert_called_once_with("old") - - @pytest.mark.parametrize("host,port", [("other-host", 1234), ("localhost", 5678)]) - def test_preserves_other_addresses(self, host, port): - self._connect(host=host, port=port) - self.worker.get_transfer_results() - assert "old" in self.worker._remote_agents - self.transport.remove_remote_agent.assert_not_called() - - def test_preserves_old_peer_when_handshake_fails(self): - self.worker.start_load_kv(self._request()) - self.worker._handshake_futures["new"].set_exception( - RuntimeError("handshake failed") - ) - assert self.worker.get_transfer_results().failed_recving == {"new-req"} - assert "old" in self.worker._remote_agents - assert self.worker._engine_by_address == {("localhost", 1234): "old"} - self.transport.remove_remote_agent.assert_not_called() - - @pytest.mark.parametrize( - "states,failed", - [ - (["PROC", "DONE"], False), - (["ERR", "PROC", "DONE"], True), - (["ERR", "PROC", "ERR"], True), - ], - ids=["healthy", "failed-sibling-completes", "failed-sibling-fails"], - ) - @pytest.mark.parametrize("invalidated", [False, True]) - def test_waits_for_reads_before_cleanup_and_failure_reporting( - self, states, failed, invalidated - ): - if invalidated: - self.nixl.invalid_remote_agents.add("old") - self.worker._recving_metadata.update(self._request("old").reqs_to_recv) - self.worker._recving_transfers["old-req"] = [1, 2] if failed else [1] - self.transport.check_xfer_state.side_effect = states - self._connect() - - result = self.worker.get_transfer_results() - assert result.finished_recving == result.failed_recving == set() - assert self.worker.get_block_ids_with_load_errors() == set() - self.transport.remove_remote_agent.assert_not_called() - - result = self.worker.get_transfer_results() - assert result.finished_recving == {"old-req"} - assert result.failed_recving == ({"old-req"} if failed else set()) - assert self.worker.get_block_ids_with_load_errors() == ( - {1} if failed else set() - ) - assert self.transport.release_xfer_handle.call_count == (2 if failed else 1) - self.transport.remove_remote_agent.assert_called_once_with("old") - assert not self.worker._invalid_remote_engines - - def test_waits_for_queued_notification(self): - metadata = self._request("old", blocks=()) - self.worker._recving_metadata.update(metadata.reqs_to_recv) - self.worker._background_nixl_handshake( - "old-req", "old", metadata.reqs_to_recv["old-req"] - ) - self._connect() - self.worker.get_transfer_results() - self.transport.remove_remote_agent.assert_not_called() - - self.worker.start_load_kv(NixlConnectorMetadata()) - self.transport.send_notif.assert_called_once_with( - "old", notif_msg=b"old-prefill:1" - ) - result = self.worker.get_transfer_results() - assert result.finished_recving == {"old-req"} - assert result.failed_recving == set() - self.transport.remove_remote_agent.assert_called_once_with("old") - - @pytest.mark.parametrize("invalidated", [False, True]) - def test_waits_for_other_handshakes(self, invalidated): - if invalidated: - self._fail_old_peer() - else: - self._connect() - self.worker._ensure_handshake("other", "other-host", 1234, 1) - self.worker.get_transfer_results() - self.transport.remove_remote_agent.assert_not_called() - self.transport.check_remote_metadata.assert_not_called() - - self._connect("other", host="other-host") - self.worker.get_transfer_results() - self.transport.remove_remote_agent.assert_called_once_with("old") - - @pytest.mark.parametrize("invalidated", [False, True]) - def test_rehandshakes_queued_request_for_released_engine(self, invalidated): - # A request queued after its engine's handshake, but drained only after - # that engine was replaced and released, must not read from it. - if invalidated: - self._fail_old_peer() - metadata = self._request("old", blocks=(), awaiting_kvs=False) - self.worker._ready_requests.put(("old-req", metadata.reqs_to_recv["old-req"])) - if not invalidated: - self._connect() - self.worker.get_transfer_results() - self.transport.remove_remote_agent.assert_called_once_with("old") - - self.worker.start_load_kv(NixlConnectorMetadata()) - assert "old" in self.worker._handshake_futures - self.transport.send_notif.assert_not_called() - - @pytest.mark.parametrize("missing", [False, True, RuntimeError("probe failed")]) - def test_recovery_requires_confirmed_missing_metadata(self, missing): - self._fail_old_peer() - # A missing rank suffices even if another rank still has metadata. - self.worker._remote_agents["old"][(0, 1)] = "old-other-rank" - self.nixl.invalid_remote_agents.clear() - if missing is True: - self.nixl.invalid_remote_agents.add("old-other-rank") - elif isinstance(missing, Exception): - self.transport.check_remote_metadata.side_effect = missing - - result = self.worker.get_transfer_results() - assert result.finished_recving == result.failed_recving == {"old-req"} - assert self.worker.get_block_ids_with_load_errors() == {1} - assert ("old" not in self.worker._remote_agents) is (missing is True) - assert self.transport.release_dlist_handle.call_count == (missing is True) - - @pytest.mark.parametrize("handshake_fails", [False, True]) - def test_recovery_rehandshakes_same_id_with_fresh_descriptors( - self, handshake_fails - ): - old_handle = self.worker.dst_xfer_side_handles["old"][0] - self._fail_old_peer() - self.worker.get_transfer_results() - self.transport.release_dlist_handle.assert_called_once_with(old_handle) - assert not self.worker._engine_by_address - with pytest.raises(KeyError): - self.worker.transfer_topo.get_engine_info("old") - self.transport.make_prepped_xfer.reset_mock() - - self.worker.start_load_kv(self._request("old", req_id="next")) - self.transport.make_prepped_xfer.assert_not_called() - if handshake_fails: - self.worker._handshake_futures["old"].set_exception( - RuntimeError("handshake failed") - ) - result = self.worker.get_transfer_results() - assert result.failed_recving == result.finished_recving == {"next"} - assert "old" not in self.worker._remote_agents - assert self.worker._ready_requests.empty() - return - - self._connect("old") - fresh_handle = self.worker.dst_xfer_side_handles["old"][0] - assert fresh_handle != old_handle - assert self.worker._engine_by_address == {("localhost", 1234): "old"} - self.transport.check_xfer_state.return_value = "DONE" - self.worker.start_load_kv(NixlConnectorMetadata()) - assert self.transport.make_prepped_xfer.call_args.args[3] == fresh_handle - result = self.worker.get_transfer_results() - assert result.finished_recving == {"next"} and not result.failed_recving - self.transport.release_dlist_handle.assert_called_once_with(old_handle) - - def test_recovery_drains_reads_and_rejects_new_ones_without_harming_other_peers( - self, - ): - self._connect("healthy", host="other-host") - self.worker._recving_metadata.update(self._request("old").reqs_to_recv) - self.worker._recving_metadata.update(self._request("healthy").reqs_to_recv) - self.worker._recving_transfers.update( - {"old-req": [10, 11], "healthy-req": [12]} - ) - self.nixl.invalid_remote_agents.add("old") - self.transport.check_xfer_state.side_effect = ( - lambda h: "ERR" if h == 10 else "PROC" - ) - old_handle = self.worker.dst_xfer_side_handles["old"][0] - - result = self.worker.get_transfer_results() - assert not result.finished_recving and not result.failed_recving - self.transport.release_dlist_handle.assert_not_called() - self.worker.start_load_kv(self._request("old", req_id="blocked")) - self.transport.make_prepped_xfer.assert_not_called() - - self.transport.check_xfer_state.side_effect = ( - lambda h: "DONE" if h == 11 else "PROC" - ) - result = self.worker.get_transfer_results() - assert ( - result.finished_recving == result.failed_recving == {"old-req", "blocked"} - ) - assert set(self.worker._remote_agents) == {"healthy"} - assert self.worker._recving_transfers == {"healthy-req": [12]} - self.transport.remove_remote_agent.assert_called_once_with("old") - calls = self.transport.mock_calls - assert calls.index(call.release_xfer_handle(11)) < calls.index( - call.release_dlist_handle(old_handle) - ) - - def test_repeated_invalidation_backs_off_before_rehandshake(self): - # A peer that is invalidated again soon after recovery (eg a persistent - # RDMA fault) must not re-handshake on every request. - self._fail_old_peer() - self.worker.get_transfer_results() - self.transport.remove_remote_agent.assert_called_once_with("old") - - self._connect("old") - self._fail_old_peer() - assert self.worker.get_transfer_results().failed_recving == {"old-req"} - assert "old" in self.worker._remote_agents - - self.transport.make_prepped_xfer.reset_mock() - self.worker.start_load_kv(self._request("old", req_id="next")) - assert self.worker.get_transfer_results().failed_recving == {"next"} - self.transport.make_prepped_xfer.assert_not_called() - assert "old" not in self.worker._handshake_futures - self.transport.remove_remote_agent.assert_called_once() - - backoff, release_at = self.worker._invalid_engine_backoff["old"] - assert backoff == 1.0 - self.worker._invalid_engine_backoff["old"] = (backoff, release_at - backoff) - self.worker.get_transfer_results() - assert self.transport.remove_remote_agent.call_count == 2 - assert "old" not in self.worker._remote_agents - - def test_recovery_waits_until_failed_handle_can_be_released(self): - self._fail_old_peer() - with patch.object( - self.transport, "release_xfer_handle", side_effect=RuntimeError("in use") - ): - result = self.worker.get_transfer_results() - assert not result.finished_recving and not result.failed_recving - assert self.worker._recving_transfers["old-req"] - self.transport.release_dlist_handle.assert_not_called() - - result = self.worker.get_transfer_results() - assert result.finished_recving == result.failed_recving == {"old-req"} - self.transport.remove_remote_agent.assert_called_once_with("old") - - @pytest.mark.parametrize("expired", [False, True]) - def test_success_and_expiry_do_not_probe_metadata(self, expired): - metadata = self._request("old") - if expired: - metadata.reqs_to_recv["old-req"].remote.blocks_expiry_time = 0.0 - self.worker._bidirectional_kv_xfer_enabled = True - with ( - patch.object(self.worker.xfer_stats, "record_failed_transfer") as failure, - patch.object(self.worker.xfer_stats, "record_kv_expired_req") as expiry, - ): - self.worker.start_load_kv(metadata) - result = self.worker.get_transfer_results() - assert result.finished_recving == {"old-req"} - assert result.failed_recving == ({"old-req"} if expired else set()) - assert expiry.call_count == int(expired) - failure.assert_not_called() - self.transport.check_remote_metadata.assert_not_called() - self.transport.remove_remote_agent.assert_not_called() - - @pytest.mark.parametrize("awaiting_kvs", [False, True]) - def test_invalid_peer_empty_recv_is_reported_only_when_awaited(self, awaiting_kvs): - self._fail_old_peer() - self.worker._recving_transfers["old-req"].append(11) - self.transport.check_xfer_state.side_effect = ( - lambda h: "PROC" if h == 11 else "ERR" - ) - self.worker.get_transfer_results() - self.transport.make_prepped_xfer.reset_mock() - - self.worker.start_load_kv( - self._request("old", blocks=(), awaiting_kvs=awaiting_kvs, req_id="empty") - ) - result = self.worker.get_transfer_results() - expected = {"empty"} if awaiting_kvs else set() - assert result.finished_recving == result.failed_recving == expected - self.transport.make_prepped_xfer.assert_not_called() - self.transport.send_notif.assert_not_called() - self.transport.remove_remote_agent.assert_not_called() - - def test_eviction_clears_pending_recovery(self): - self.worker._failed_remote_engines.add("old") - self.worker._invalid_remote_engines.add("old") - self.worker._engine_ttl = 1 - self.worker._engine_last_active["old"] = time.perf_counter() - 10 - self.worker._evict_stale_engines() - self.worker.get_transfer_results() - self.transport.remove_remote_agent.assert_called_once_with("old") - assert not self.worker._failed_remote_engines - assert not self.worker._invalid_remote_engines - - -def test_transfer_topology_unregister(): - """TransferTopology.unregister_remote_engine removes the engine.""" - from vllm.v1.attention.backends.flash_attn import FlashAttentionBackend - - topo = TransferTopology( - tp_rank=0, - tp_size=1, - block_size=16, - engine_id="local", - is_mla=False, - is_mamba=False, - total_num_kv_heads=4, - attn_backends=[FlashAttentionBackend], - ) - - info = EngineTransferInfo( - remote_tp_size=1, - remote_block_size=16, - remote_block_len=64, - remote_physical_blocks_per_logical=1, - ) - topo.register_remote_engine("remote-1", info) - assert topo.get_engine_info("remote-1") is info - - topo.unregister_remote_engine("remote-1") - with pytest.raises(KeyError): - topo.get_engine_info("remote-1") - - # Idempotent: no error on double-unregister - topo.unregister_remote_engine("remote-1") - - -@patch( - "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper", - FakeNixlWrapper, -) -def test_aborted_request_removed_from_worker_in_batch(default_vllm_config, dist_init): - """Create and schedule a request so that P adds it to in-batch tracking via - the real scheduler, then simulate an abort (request not in next scheduler - iteration) and verify the worker no longer tracks it as in-batch. - """ - block_size = maybe_update_block_size(16) - vllm_config = create_vllm_config(block_size=block_size) - - scheduler = create_scheduler(vllm_config) - # KVConnector Worker in P - connector = NixlConnector( - vllm_config, - KVConnectorRole.WORKER, - make_kv_cache_config(block_size=block_size), - ) - connector.connector_worker = FakeNixlConnectorWorker( - vllm_config, connector.engine_id, hand_shake_latency=0 - ) - - # Create a request that triggers do_remote_decode so that - # the scheduler adds it to reqs_in_batch - req = create_request(request_id=1, do_remote_decode=True, max_tokens=1) - scheduler.add_request(req) - - # First scheduling pass - examine build_connector_meta output - sched_out = scheduler.schedule() - kv_meta = sched_out.kv_connector_metadata - assert kv_meta is not None - assert isinstance(kv_meta, NixlConnectorMetadata) - assert req.request_id in kv_meta.reqs_in_batch - - #### Model Runner start #### - # Bind scheduler-produced metadata and start worker processing. - connector.bind_connector_metadata(kv_meta) - - dummy_ctx = ForwardContext( - no_compile_layers={}, - attn_metadata={}, - slot_mapping={}, - ) - connector.start_load_kv(dummy_ctx) - - # Ensure it was tracked by the worker - assert req.request_id in connector.connector_worker._reqs_to_process - - #### Model Runner end #### - - # Abort request - request_finished call in connector scheduler - scheduler.finish_requests(req.request_id, RequestStatus.FINISHED_ABORTED) - # Second scheduling pass - build metadata with aborted request - sched_out2 = scheduler.schedule() - kv_meta2 = sched_out2.kv_connector_metadata - assert kv_meta2 is not None - assert isinstance(kv_meta2, NixlConnectorMetadata) - assert req.request_id not in kv_meta2.reqs_in_batch - - # Bind empty/abort metadata and run worker step - #### Model Runner start #### - connector.bind_connector_metadata(kv_meta2) - connector.start_load_kv(dummy_ctx) - - # After abort, the worker should not keep tracking it as "in-batch" - assert req.request_id not in connector.connector_worker._reqs_to_process - #### Model Runner end #### - - -class FailingNixlWrapper(FakeNixlWrapper): - """Mock NixlWrapper that fails on specific operations.""" - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.fail_handshake = False - self.fail_transfer_setup = False - self.fail_send_notif = False - self.fail_transfer_state = False # Returns "ERR" state - self.fail_transfer_exception = False # Raises exception in check_xfer_state - - def add_remote_agent(self, agent_metadata: bytes) -> str: - if self.fail_handshake: - from zmq.error import Again - - raise Again("Simulated timeout failure") - return super().add_remote_agent(agent_metadata) - - def make_prepped_xfer( - self, - xfer_type: str, - local_xfer_side_handle: int, - local_block_descs_ids: list[int], - remote_xfer_side_handle: int, - remote_block_descs_ids: list[int], - notif_msg: bytes | None = None, - ) -> int: - if self.fail_transfer_setup: - # classic RuntimeError to simulate failure - raise RuntimeError("BAD STATUS") - return super().make_prepped_xfer( - xfer_type, - local_xfer_side_handle, - local_block_descs_ids, - remote_xfer_side_handle, - remote_block_descs_ids, - notif_msg, - ) - - def send_notif(self, agent_name: str, notif_msg: bytes) -> None: - if self.fail_send_notif: - raise RuntimeError("Simulated send_notif failure") - return super().send_notif(agent_name, notif_msg) - - def check_xfer_state(self, handle: int) -> str: - if self.fail_transfer_exception: - raise RuntimeError("Simulated check_xfer_state exception") - if self.fail_transfer_state: - return "ERR" # Bad transfer state - return super().check_xfer_state(handle) - - -@patch( - "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper", - FailingNixlWrapper, -) -@pytest.mark.parametrize("awaiting_kvs", [True, False]) -@pytest.mark.parametrize("notification_fails", [False, True]) -def test_empty_recv_is_reported_only_when_awaited( - default_vllm_config, dist_init, awaiting_kvs, notification_fails -): - """An empty recv is reported iff the scheduler parked the request on it. - - _read_blocks returns early when the local block list is empty -- D already - holds the KV and only the producer needs telling. Two kinds of caller reach - that path, and they need opposite handling: - - Parked (awaiting_kvs=True, a full prefix cache hit). Without a report the - request is named in neither _recving_transfers nor _failed_recv_reqs and - never reaches finished_recving. The scheduler has no other way to release a - WAITING_FOR_REMOTE_KVS request, and there is no timeout, so it sits in - kv_holding_waiting holding its blocks for the life of the process. - - Notify-only (awaiting_kvs=False): request_finished seeding an empty recv to - free P's blocks for a request aborted before it was scheduled, or a readback - on a request that stays RUNNING. Reporting either one crashes the engine - core -- _update_from_kv_xfer_finished asserts the id is still in - self.requests and that the request is parked or finished. - - A failed notification changes neither: the KV is local either way, and the - producer frees its own blocks on a timeout. - """ - block_size = maybe_update_block_size(16) - vllm_config = create_vllm_config(block_size=block_size) - connector = NixlConnector( - vllm_config, - KVConnectorRole.WORKER, - make_kv_cache_config(block_size=block_size), - ) - connector.connector_worker = FakeNixlConnectorWorker( - vllm_config, - connector.engine_id, - hand_shake_latency=0.0, - kv_cache_config=connector._kv_cache_config, - ) - connector.connector_worker.nixl_wrapper.fail_send_notif = notification_fails - - request_id = f"test_empty_recv_awaited_{awaiting_kvs}" - metadata = NixlConnectorMetadata() - metadata.add_new_req_to_recv( - request_id=request_id, - local_block_ids=(), # empty: the whole prompt is already cached locally - kv_transfer_params={ - "remote_block_ids": [[20, 21, 22]], - "remote_engine_id": FakeNixlConnectorWorker.REMOTE_ENGINE_ID, - "remote_request_id": f"prefill-{request_id}", - "remote_host": "localhost", - "remote_port": 1234, - "remote_tp_size": 1, - }, - awaiting_kvs=awaiting_kvs, - ) - connector.bind_connector_metadata(metadata) - dummy_ctx = ForwardContext(no_compile_layers={}, attn_metadata={}, slot_mapping={}) - connector.start_load_kv(dummy_ctx) - connector.bind_connector_metadata(NixlConnectorMetadata()) - time.sleep(0.1) - connector.start_load_kv(dummy_ctx) - - _, done_recving = connector.get_finished(finished_req_ids=set()) - assert (request_id in done_recving) is awaiting_kvs - - -@pytest.mark.parametrize("is_hma", [False, True]) -@pytest.mark.parametrize( - "local_block_ids,awaiting_kvs", - [((), False), (([],), False), (([],), True), (([1, 2, 3],), True)], -) -def test_handshake_failure_reports_only_awaited_recvs( - recv_worker, is_hma, local_block_ids, awaiting_kvs -): - """A failed cleanup handshake must not complete an already-aborted request.""" - worker = recv_worker - worker._is_hma_required = is_hma - worker._recving_metadata.clear() - worker._remote_agents = {} - worker._handshake_lock = contextlib.nullcontext() - worker._ready_requests = queue.Queue() - worker._reqs_to_process = set() - worker.pcp_rank = 0 - metadata = NixlConnectorMetadata() - metadata.add_new_req_to_recv( - request_id="request", - local_block_ids=local_block_ids, - kv_transfer_params={ - "remote_block_ids": ([4, 5, 6],), - "remote_engine_id": "prefill", - "remote_request_id": "prefill-request", - "remote_host": "localhost", - "remote_port": 1234, - }, - awaiting_kvs=awaiting_kvs, - ) - handshake = Future[None]() - handshake.set_exception(RuntimeError("handshake failed")) - with patch.object(worker, "_ensure_handshake", return_value=handshake): - worker.start_load_kv(metadata) - - results = worker.get_transfer_results() - expected = {"request"} if awaiting_kvs else set() - assert results.finished_recving == results.failed_recving == expected - assert worker.get_block_ids_with_load_errors() == ( - set(local_block_ids[0]) if awaiting_kvs and not is_hma else set() - ) - assert not worker._recving_metadata - assert not worker._recv_failures - assert not worker._pending_recv_notifs - - -@patch( - "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper", - FailingNixlWrapper, -) -@pytest.mark.parametrize( - "failure_type,wrapper_config,needs_get_finished", - [ - ("transfer_setup_failed", {"fail_transfer_setup": True}, False), - ("handshake_failed", {"fail_handshake": True}, False), - ("notification_failed", {"fail_send_notif": True}, False), - ("transfer_failed", {"fail_transfer_state": True}, True), - ("transfer_exception", {"fail_transfer_exception": True}, True), - ], -) -@pytest.mark.parametrize("enable_hma", [False, True]) -def test_transfer_failure_logging( - default_vllm_config, - dist_init, - failure_type, - wrapper_config, - needs_get_finished, - enable_hma, -): - """Test that transfer failures are logged with structured context. - - Run with `pytest -sv` to see the log output. - - Covers failure types: - - transfer_setup_failed: make_prepped_xfer fails - - handshake_failed: add_remote_agent fails during request handshake - - notification_failed: send_notif fails - - transfer_failed: check_xfer_state returns bad state (e.g., "ERR") - - transfer_exception: check_xfer_state raises exception - """ - import logging - - block_size = maybe_update_block_size(16) - vllm_config = create_vllm_config(block_size=block_size) - - connector = NixlConnector( - vllm_config, - KVConnectorRole.WORKER, - make_kv_cache_config(block_size=block_size, swa_enabled=enable_hma), - ) - connector.connector_worker = FakeNixlConnectorWorker( - vllm_config, - connector.engine_id, - hand_shake_latency=0.0, - kv_cache_config=connector._kv_cache_config, - ) - - # Configure FailingNixlWrapper to fail in the specified way - for key, value in wrapper_config.items(): - setattr(connector.connector_worker.nixl_wrapper, key, value) - - request_id = f"test_{failure_type}_req" - - # For notification_failed, we need empty local blocks - # (full cache hit path to trigger send_notif) - local_blocks: tuple[()] | tuple[list[int], ...] - if enable_hma: - # HMA enabled: multiple groups (FA + SW) - local_blocks = ( - () if failure_type == "notification_failed" else ([10, 11, 12], [13, 14]) - ) - remote_blocks = [[20, 21, 22], [23, 24]] - else: - # HMA disabled: single group - local_blocks = () if failure_type == "notification_failed" else ([10, 11, 12],) - remote_blocks = [[20, 21, 22]] - - metadata = NixlConnectorMetadata() - metadata.add_new_req_to_recv( - request_id=request_id, - local_block_ids=local_blocks, - kv_transfer_params={ - "remote_block_ids": remote_blocks, - "remote_engine_id": FakeNixlConnectorWorker.REMOTE_ENGINE_ID, - "remote_request_id": f"prefill-{request_id}", - "remote_host": "localhost", - "remote_port": 1234, - "remote_tp_size": 1, - }, - ) - connector.bind_connector_metadata(metadata) - - dummy_ctx = ForwardContext( - no_compile_layers={}, - attn_metadata={}, - slot_mapping={}, - ) - - # Capture logs from the nixl connector loggers - # vLLM loggers have propagate=False, so we need to capture directly - nixl_logger = logging.getLogger( - "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker" - ) - pull_logger = logging.getLogger( - "vllm.distributed.kv_transfer.kv_connector.v1.nixl.pull_worker" - ) - captured_logs: list[logging.LogRecord] = [] - - class LogCapture(logging.Handler): - def emit(self, record): - captured_logs.append(record) - - handler = LogCapture() - handler.setLevel(logging.ERROR) - nixl_logger.addHandler(handler) - pull_logger.addHandler(handler) - - try: - connector.start_load_kv(dummy_ctx) - # Process the ready_requests queue (for async handshake) - connector.bind_connector_metadata(NixlConnectorMetadata()) - # Wait for async handshake to complete - time.sleep(0.2) - connector.start_load_kv(dummy_ctx) - - # For transfer_failed/transfer_exception, the error happens in - # get_finished() when checking transfer state - if needs_get_finished: - connector.get_finished(finished_req_ids=set()) - finally: - nixl_logger.removeHandler(handler) - pull_logger.removeHandler(handler) - - # Print logs for manual comparison between commits - error_logs = [r for r in captured_logs if r.levelno >= logging.ERROR] - print("\n" + "=" * 60) - print(f"CAPTURED ERROR LOGS for {failure_type}:") - print("=" * 60) - for i, record in enumerate(error_logs): - print(f"\n--- Log {i + 1} ---") - print(f"Message: {record.message}") - print("=" * 60 + "\n") - - assert len(error_logs) >= 1, f"Expected at least one error log for {failure_type}" - - # Verify structured logging output (new format) - # Check that at least one log matches the expected format - all_messages = [r.message for r in error_logs] - combined_logs = "\n".join(all_messages) - - assert any("NIXL transfer failure" in msg for msg in all_messages), ( - f"Expected structured log format with 'NIXL transfer failure' prefix " - f"for {failure_type}. Got: {all_messages}" - ) - assert any("failure_type" in msg for msg in all_messages), ( - f"Expected 'failure_type' in logs. Got: {all_messages}" - ) - assert any("Context:" in msg for msg in all_messages), ( - f"Expected 'Context:' in logs. Got: {all_messages}" - ) - # Check that the expected failure_type appears in at least one log - # Note: handshake_failed also triggers handshake_setup_failed - assert failure_type in combined_logs or ( - failure_type == "handshake_failed" and "handshake_setup_failed" in combined_logs - ), f"Expected '{failure_type}' in logs. Got: {all_messages}" - - -@patch( - "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper", - FailingNixlWrapper, -) -def test_handshake_failure_returns_finished(default_vllm_config, dist_init): - """Test that handshake failures mark blocks invalid and return via get_finished.""" - block_size = maybe_update_block_size(16) - vllm_config = create_vllm_config(block_size=block_size) - - connector = NixlConnector( - vllm_config, - KVConnectorRole.WORKER, - make_kv_cache_config(block_size=block_size), - ) - connector.connector_worker = FakeNixlConnectorWorker( - vllm_config, connector.engine_id, hand_shake_latency=0.1 - ) - connector.connector_worker.nixl_wrapper.fail_handshake = True - - request_id = "test_handshake_fail" - metadata = NixlConnectorMetadata() - metadata.add_new_req_to_recv( - request_id=request_id, - local_block_ids=([1, 2, 3],), - kv_transfer_params={ - "remote_block_ids": ([4, 5, 6],), - "remote_engine_id": FakeNixlConnectorWorker.REMOTE_ENGINE_ID, - "remote_request_id": f"prefill-{request_id}", - "remote_host": "localhost", - "remote_port": 1234, - "remote_tp_size": 1, - }, - ) - connector.bind_connector_metadata(metadata) - - dummy_ctx = ForwardContext( - no_compile_layers={}, - attn_metadata={}, - slot_mapping={}, - ) - connector.start_load_kv(dummy_ctx) - - # Wait for handshake to fail - time.sleep(0.3) - - assert connector.get_block_ids_with_load_errors() == set() - _, done_recving = connector.get_finished(finished_req_ids=set()) - assert request_id in done_recving - assert connector.get_block_ids_with_load_errors() == {1, 2, 3} - - # Handshake failures are recorded as transport failures, separately - # from KV expiry. - assert connector.connector_worker.xfer_stats.data["num_failed_handshakes"] - assert connector.connector_worker.xfer_stats.data["num_failed_transfers"] == [] - - -@patch( - "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper", - FailingNixlWrapper, -) -@pytest.mark.parametrize("is_hma", [False, True]) -def test_transfer_setup_failure_returns_finished( - default_vllm_config, dist_init, is_hma -): - """Setup failures report the request; only non-HMA reports block IDs.""" - block_size = maybe_update_block_size(16) - vllm_config = create_vllm_config(block_size=block_size) - - connector = NixlConnector( - vllm_config, - KVConnectorRole.WORKER, - make_kv_cache_config(block_size=block_size), - ) - connector.connector_worker = FakeNixlConnectorWorker( - vllm_config, connector.engine_id, hand_shake_latency=0 - ) - connector.connector_worker.nixl_wrapper.fail_transfer_setup = True - connector.connector_worker._is_hma_required = is_hma - - request_id = "test_transfer_fail" - metadata = NixlConnectorMetadata() - metadata.add_new_req_to_recv( - request_id=request_id, - local_block_ids=([7, 8, 9],), - kv_transfer_params={ - "remote_block_ids": ([10, 11, 12],), - "remote_engine_id": FakeNixlConnectorWorker.REMOTE_ENGINE_ID, - "remote_request_id": f"prefill-{request_id}", - "remote_host": "localhost", - "remote_port": 1234, - "remote_tp_size": 1, - }, - ) - connector.bind_connector_metadata(metadata) - - dummy_ctx = ForwardContext( - no_compile_layers={}, - attn_metadata={}, - slot_mapping={}, - ) - connector.start_load_kv(dummy_ctx) - - # Wait for handshake to complete and process ready_requests - connector.bind_connector_metadata(NixlConnectorMetadata()) - time.sleep(0.1) - connector.start_load_kv(dummy_ctx) - - assert connector.get_block_ids_with_load_errors() == set() - results = connector.get_transfer_results(finished_req_ids=set()) - assert request_id in results.finished_recving - assert results.failed_recving == {request_id} - invalid_blocks = connector.get_block_ids_with_load_errors() - assert invalid_blocks == (set() if is_hma else {7, 8, 9}) - - -class _ScriptedXferWrapper(FakeNixlWrapper): - """Scripts per-handle xfer states; forbids releasing in-flight handles.""" - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - # handle -> successive states; the last one repeats. - self.xfer_states: dict[int, list[str]] = {} - self.released: list[int] = [] - - def check_xfer_state(self, handle: int) -> str: - states = self.xfer_states[handle] - return states.pop(0) if len(states) > 1 else states[0] - - def release_xfer_handle(self, handle: int) -> None: - # A posted-but-unfinished transfer cannot be aborted: releasing it - # leaves the RDMA READ armed. Production code must never do this. - assert self.xfer_states.get(handle, ["DONE"])[0] != "PROC", ( - f"released in-flight handle {handle}" - ) - self.released.append(handle) - - -def _make_split_read_connector(vllm_config, request_id, states): - """Seed a request with scripted in-flight xfer handles + recv metadata.""" - connector = NixlConnector( - vllm_config, - KVConnectorRole.WORKER, - make_kv_cache_config(block_size=vllm_config.cache_config.block_size), - ) - connector.connector_worker = FakeNixlConnectorWorker( - vllm_config, connector.engine_id, hand_shake_latency=0 - ) - worker = connector.connector_worker - wrapper = _ScriptedXferWrapper("agent") - worker.nixl_wrapper = wrapper - metadata = NixlConnectorMetadata() - metadata.add_new_req_to_recv( - request_id=request_id, - local_block_ids=([7, 8, 9],), - kv_transfer_params={ - "remote_block_ids": ([10, 11, 12],), - "remote_engine_id": FakeNixlConnectorWorker.REMOTE_ENGINE_ID, - "remote_request_id": f"prefill-{request_id}", - "remote_host": "localhost", - "remote_port": 1234, - "remote_tp_size": 1, - }, - ) - worker._recving_metadata[request_id] = metadata.reqs_to_recv[request_id] - wrapper.xfer_states = dict(states) - worker._recving_transfers[request_id] = list(states) - return connector, worker, wrapper - - -@patch( - "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper", - FakeNixlWrapper, -) -def test_split_read_failure_defers_report_until_last_handle( - default_vllm_config, dist_init -): - """One half of a split (mixed DRAM/VRAM) read failing must not report the - request — nor invalidate its blocks — while the sibling xfer is still in - flight: a posted READ cannot be aborted and would DMA into blocks the - scheduler could free and reuse. The report happens exactly once, when the - last handle is terminal.""" - request_id = "split_read_partial_failure" - err_handle, live_handle = 11, 22 - connector, worker, wrapper = _make_split_read_connector( - create_vllm_config(block_size=maybe_update_block_size(16)), - request_id, - {err_handle: ["ERR"], live_handle: ["PROC", "PROC", "DONE"]}, - ) - - # Poll 1: one half fails; the sibling is in flight -> nothing reported. - _, done_recving = connector.get_finished(finished_req_ids=set()) - assert done_recving == set() - assert connector.get_block_ids_with_load_errors() == set() - assert request_id in worker._recving_metadata - assert wrapper.released == [err_handle] - - # Poll 2: sibling still in flight. - _, done_recving = connector.get_finished(finished_req_ids=set()) - assert done_recving == set() - - # Poll 3: sibling terminal -> reported exactly once, blocks invalidated. - _, done_recving = connector.get_finished(finished_req_ids=set()) - assert done_recving == {request_id} - assert connector.get_block_ids_with_load_errors() == {7, 8, 9} - assert request_id not in worker._recving_metadata - assert wrapper.released == [err_handle, live_handle] - - # Poll 4: nothing left; no double report. - _, done_recving = connector.get_finished(finished_req_ids=set()) - assert done_recving == set() - - -@patch( - "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper", - FailingNixlWrapper, -) -@pytest.mark.parametrize( - "failure_mode", - [ - "handshake", - "transfer_setup", - "transfer_failed", - "transfer_exception", - ], -) -def test_failed_request_skips_kv_postprocessing( - default_vllm_config, dist_init, failure_mode -): - """Test that failed requests skip KV sync and post-processing in - get_transfer_results(). - - This is the core safety behavior: when a KV transfer fails at any stage, - the request must still appear in done_recving (so the scheduler can apply - kv_load_failure_policy), but sync_recved_kv_to_device and post-processing - must NOT be called since no valid KV data was received. - - Covers all failure paths that involve an actual (attempted) KV transfer: - - handshake: add_remote_agent raises during async handshake - - transfer_setup: make_prepped_xfer raises before handle is in transfers - - transfer_failed: check_xfer_state returns bad state ("ERR") in - _pop_done_transfers — this is the path that previously had the bug - where post-processing was NOT skipped - - transfer_exception: check_xfer_state raises in _pop_done_transfers - - Note: notification_failed (send_notif raises on the full-cache-hit path) - is intentionally excluded. That path is a best-effort D→P courtesy - notification; the blocks are already in D's cache, so no KV transfer - was attempted and done_recving is correctly empty. - """ - # Map each failure mode to the FailingNixlWrapper attribute to set. - _WRAPPER_CONFIG: dict[str, str] = { - "handshake": "fail_handshake", - "transfer_setup": "fail_transfer_setup", - "transfer_failed": "fail_transfer_state", - "transfer_exception": "fail_transfer_exception", - } - - # Use enable_permute_local_kv=True so that - # post_process_device_kv_on_receive would be called on the success path, - # making the assertion meaningful (not trivially true). - block_size = maybe_update_block_size(16) - vllm_config = create_vllm_config( - enable_permute_local_kv=True, block_size=block_size - ) - - connector = NixlConnector( - vllm_config, - KVConnectorRole.WORKER, - make_kv_cache_config(block_size=block_size), - ) - connector.connector_worker = FakeNixlConnectorWorker( - vllm_config, - connector.engine_id, - hand_shake_latency=0.1 if failure_mode == "handshake" else 0, - ) - worker = connector.connector_worker - setattr(worker.nixl_wrapper, _WRAPPER_CONFIG[failure_mode], True) - - request_id = f"test_{failure_mode}_skip_postprocess" - metadata = NixlConnectorMetadata() - metadata.add_new_req_to_recv( - request_id=request_id, - local_block_ids=([1, 2, 3],), - kv_transfer_params={ - "remote_block_ids": ([4, 5, 6],), - "remote_engine_id": FakeNixlConnectorWorker.REMOTE_ENGINE_ID, - "remote_request_id": f"prefill-{request_id}", - "remote_host": "localhost", - "remote_port": 1234, - "remote_tp_size": 1, - }, - ) - connector.bind_connector_metadata(metadata) - - dummy_ctx = ForwardContext( - no_compile_layers={}, - attn_metadata={}, - slot_mapping={}, - ) - connector.start_load_kv(dummy_ctx) - - if failure_mode == "handshake": - # Wait for async handshake to fail. - time.sleep(0.3) - else: - # All other modes: let the handshake complete, then process the - # ready_requests queue. For transfer_failed / transfer_exception the - # handle ends up in _recving_transfers; the failure surfaces in - # get_finished() via _pop_done_transfers below. - connector.bind_connector_metadata(NixlConnectorMetadata()) - time.sleep(0.1) - connector.start_load_kv(dummy_ctx) - - # Spy on sync_recved_kv_to_device and post_process_device_kv_on_receive - # to verify they are NOT called for the failed request. - with ( - patch.object(worker, "sync_recved_kv_to_device") as mock_sync, - patch.object(worker, "post_process_device_kv_on_receive") as mock_postprocess, - ): - results = connector.get_transfer_results(finished_req_ids=set()) - - assert request_id in results.finished_recving - assert results.failed_recving == {request_id} - - # Critical: KV sync and post-processing must NOT have been called - # since no valid KV data was received for the failed request. - mock_sync.assert_not_called() - mock_postprocess.assert_not_called() - - # Metadata for the request should have been cleaned up. - assert request_id not in worker._recving_metadata - - # Blocks should have been marked as invalid. - invalid_blocks = connector.get_block_ids_with_load_errors() - assert invalid_blocks == {1, 2, 3} - - -@pytest.mark.parametrize("sibling_state", ["ERR", "DONE"]) -def test_recv_failure_waits_for_sibling_transfer(recv_worker, sibling_state): - """Failure must not release blocks while a sibling can still write to them.""" - worker = recv_worker - worker._recving_transfers["request"] = [101, 102] - worker._pending_recv_notifs = {"request": [("prefill", b"request:1")]} - worker.nixl_wrapper.check_xfer_state.side_effect = ["ERR", "PROC", sibling_state] - - results = worker.get_transfer_results() - assert results.finished_recving == results.failed_recving == set() - assert worker.get_block_ids_with_load_errors() == set() - assert "request" in worker._recving_metadata - worker.nixl_wrapper.release_xfer_handle.assert_called_once_with(101) - - results = worker.get_transfer_results() - assert results.finished_recving == results.failed_recving == {"request"} - assert worker.get_block_ids_with_load_errors() == {1, 2, 3} - worker.nixl_wrapper.send_notif.assert_not_called() - - results = worker.get_transfer_results() - assert results.finished_recving == results.failed_recving == set() - assert worker.get_block_ids_with_load_errors() == set() - - -def test_recv_failure_waits_for_unpollable_handle_to_be_released(recv_worker): - """A polling exception and failed release do not prove that DMA has stopped.""" - worker = recv_worker - worker._recving_transfers["request"] = [101] - worker.nixl_wrapper.check_xfer_state.side_effect = [RuntimeError("poll"), "DONE"] - worker.nixl_wrapper.release_xfer_handle.side_effect = [ - RuntimeError("release"), - None, - ] - - results = worker.get_transfer_results() - assert results.finished_recving == results.failed_recving == set() - assert worker.get_block_ids_with_load_errors() == set() - assert "request" in worker._recving_metadata - - results = worker.get_transfer_results() - assert results.finished_recving == results.failed_recving == {"request"} - assert worker.get_block_ids_with_load_errors() == {1, 2, 3} - assert worker.nixl_wrapper.release_xfer_handle.call_count == 2 - - -def _set_test_speculative_config( - vllm_config, - *, - method: str = "eagle3", - model: str = "test/eagle3-drafter", - revision: str | None = None, - code_revision: str | None = None, - num_speculative_tokens: int = 1, - parallel_drafting: bool = False, - kv_cache_dtype: str | None = None, - attention_backend: str | None = None, - auxiliary_layer_ids: tuple[int, ...] = (2, 16, 29), -) -> None: - draft_model_config = SimpleNamespace( - model=model, - revision=revision, - code_revision=code_revision, - hf_config=SimpleNamespace(eagle_aux_hidden_state_layer_ids=auxiliary_layer_ids), - ) - vllm_config.speculative_config = SimpleNamespace( - method=method, - draft_model_config=draft_model_config, - num_speculative_tokens=num_speculative_tokens, - parallel_drafting=parallel_drafting, - kv_cache_dtype=kv_cache_dtype, - attention_backend=attention_backend, - use_eagle=lambda: True, - ) - - -@pytest.mark.parametrize( - "remote_overrides,should_match", - [ - ({}, True), - ({"num_speculative_tokens": 2}, True), - ({"method": "mtp"}, False), - ({"model": "test/different-drafter"}, False), - ({"revision": "different-revision"}, False), - ({"parallel_drafting": True}, False), - ({"kv_cache_dtype": "fp8"}, False), - # attention_backend is intentionally not part of the compat hash - # (see _get_speculative_compatibility_factors); overriding it must - # not change the hash. - ({"attention_backend": "FLASHINFER"}, True), - ({"auxiliary_layer_ids": (2, 16, 30)}, False), - ], -) -@pytest.mark.skip_global_cleanup -def test_speculative_config_compatibility_hash( - remote_overrides: dict[str, Any], should_match: bool -): - local_config = create_vllm_config() - remote_config = create_vllm_config() - _set_test_speculative_config(local_config) - _set_test_speculative_config(remote_config, **remote_overrides) - - local_hash = compute_nixl_compatibility_hash(local_config, "FLASH_ATTN") - remote_hash = compute_nixl_compatibility_hash(remote_config, "FLASH_ATTN") - - assert (local_hash == remote_hash) is should_match - - -@pytest.mark.skip_global_cleanup -def test_missing_speculative_config_changes_compatibility_hash(): - regular_config = create_vllm_config() - speculative_config = create_vllm_config() - _set_test_speculative_config(speculative_config) - - regular_hash = compute_nixl_compatibility_hash(regular_config, "FLASH_ATTN") - speculative_hash = compute_nixl_compatibility_hash(speculative_config, "FLASH_ATTN") - - assert regular_hash != speculative_hash - - -@pytest.mark.skip_global_cleanup -def test_speculative_kv_cache_dtype_resolves_to_target(): - # The draft kv_cache_dtype override defaults to None ("inherit the target's - # --kv-cache-dtype"). An explicit setting on one side that matches the - # other side's inherited (resolved) dtype must not spuriously mismatch. - local_config = create_vllm_config(cache_dtype="fp8") - remote_config = create_vllm_config(cache_dtype="fp8") - _set_test_speculative_config(local_config, kv_cache_dtype="fp8") # explicit - _set_test_speculative_config(remote_config, kv_cache_dtype=None) # inherits - - local_hash = compute_nixl_compatibility_hash(local_config, "FLASH_ATTN") - remote_hash = compute_nixl_compatibility_hash(remote_config, "FLASH_ATTN") - - assert local_hash == remote_hash - - -@pytest.mark.skip_global_cleanup -def test_speculative_attention_backend_not_in_compatibility_hash(): - # The draft attention_backend is intentionally excluded from the hash: the - # connector only has the raw override (auto-select), and its transfer- - # relevant effect is validated per region at runtime. Differing overrides - # must not change the hash. - local_config = create_vllm_config() - remote_config = create_vllm_config() - _set_test_speculative_config(local_config, attention_backend=None) - _set_test_speculative_config(remote_config, attention_backend="FLASHINFER") - - local_hash = compute_nixl_compatibility_hash(local_config, "FLASH_ATTN") - remote_hash = compute_nixl_compatibility_hash(remote_config, "FLASH_ATTN") - - assert local_hash == remote_hash - - -@pytest.mark.skip_global_cleanup -def test_transfer_mode_changes_compatibility_hash(): - # push (WRITE) and pull (READ) connectors use incompatible transfer - # protocols, so their compatibility hashes must differ; identical modes - # must match. The default mode is pull. - config = create_vllm_config() - - pull_hash = compute_nixl_compatibility_hash( - config, "FLASH_ATTN", transfer_mode="pull" - ) - push_hash = compute_nixl_compatibility_hash( - config, "FLASH_ATTN", transfer_mode="push" - ) - - assert pull_hash != push_hash - assert pull_hash == compute_nixl_compatibility_hash( - config, "FLASH_ATTN", transfer_mode="pull" - ) - assert compute_nixl_compatibility_hash(config, "FLASH_ATTN") == pull_hash - - -@pytest.mark.skip_global_cleanup -@pytest.mark.parametrize("mode", ["pull", "push"]) -@pytest.mark.parametrize( - "tp_size,pcp_size,dcp_size,transfer_tp_size", - [(1, 1, 1, 1), (4, 1, 1, 4), (4, 1, 4, 4), (1, 4, 1, 1), (1, 4, 4, 4)], -) -def test_scheduler_advertises_transfer_topology( - mode, tp_size, pcp_size, dcp_size, transfer_tp_size -): - """The consumer must address every distinct producer KV shard.""" - from vllm.distributed.kv_transfer.kv_connector.v1.nixl.pull_scheduler import ( - NixlPullConnectorScheduler, - ) - from vllm.distributed.kv_transfer.kv_connector.v1.nixl.push_scheduler import ( - NixlPushConnectorScheduler, - ) - - config = create_vllm_config(kv_role="kv_producer") - config.parallel_config.tensor_parallel_size = tp_size - config.parallel_config.prefill_context_parallel_size = pcp_size - config.parallel_config.decode_context_parallel_size = dcp_size - cls = NixlPullConnectorScheduler if mode == "pull" else NixlPushConnectorScheduler - scheduler = cls(config, "prefiller", make_kv_cache_config(block_size=16)) - request = create_request(request_id=1, num_tokens=32, do_remote_decode=True) - request.status = RequestStatus.FINISHED_LENGTH_CAPPED - try: - delay, params = scheduler.request_finished(request, ([0, 1],)) - assert delay - assert params["transfer_mode"] == mode - assert params["tp_size"] == transfer_tp_size - assert params["dcp_size"] == dcp_size - finally: - scheduler.shutdown() - - -@pytest.mark.parametrize( - "mismatch_type,config_overrides,version_override,should_fail,enforce_handshake_compat", - [ - ("vllm_version", {}, {"vllm_version": "0.6.1"}, True, True), - ("nixl_connector_version", {}, {"connector_version": 37}, True, True), - ("model_name", {"model": "facebook/opt-350m"}, {}, True, True), - ("dtype", {"dtype": "bfloat16"}, {}, True, True), - ("cache_dtype", {"cache_dtype": "fp8"}, {}, True, True), - ("num_kv_heads", {"hf_overrides": {"num_key_value_heads": 8}}, {}, True, True), - ( - "num_hidden_layers", - {"hf_overrides": {"num_hidden_layers": 24}}, - {}, - True, - True, - ), - ("hidden_size", {"hf_overrides": {"hidden_size": 1536}}, {}, True, True), - ("block_size", {"block_size": 8}, {}, False, True), - ("matching_config", {}, {}, False, True), - ("escape_hatch", {"model": "facebook/opt-350m"}, {}, False, False), - ], -) -@patch( - "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper", - FakeNixlWrapper, -) -def test_compatibility_hash_validation( - default_vllm_config, - dist_init, - mismatch_type, - config_overrides, - version_override, - should_fail, - enforce_handshake_compat, -): - """Test NIXL compatibility hash validation during handshake. - - Parameters - ---------- - mismatch_type: description of what is being tested - config_overrides: dict of config to override for the remote instance - version_override: version dict e.g. {"vllm_version": "0.6.1"} - should_fail: whether the handshake should fail - enforce_handshake_compat: whether to enforce compatibility checking - - """ - block_size = maybe_update_block_size(16) - local_vllm_config = create_vllm_config( - model="facebook/opt-125m", - block_size=block_size, - kv_connector_extra_config={ - "enforce_handshake_compat": enforce_handshake_compat - }, - ) - kv_cache_config = make_kv_cache_config(block_size=block_size, num_blocks=2) - decode_connector = NixlConnector( - local_vllm_config, KVConnectorRole.WORKER, kv_cache_config - ) - decode_worker = decode_connector.connector_worker - kv_cache_spec = cast( - AttentionSpec, kv_cache_config.kv_cache_groups[0].kv_cache_spec - ) - shape = compute_layer_kv_cache_shape_bytes( - kv_cache_spec, kv_cache_config.num_blocks - ) - shared_tensor = torch.zeros(*shape, dtype=torch.int8).view(kv_cache_spec.dtype) - unique_tensor = torch.zeros(*shape, dtype=torch.int8).view(kv_cache_spec.dtype) - # Build kv_caches from the actual layer names in kv_cache_config so that - # _layer_specs lookups in register_kv_caches always find a matching key. - layer_names = [ - name for group in kv_cache_config.kv_cache_groups for name in group.layer_names - ] - kv_caches = { - name: shared_tensor if i % 2 == 0 else unique_tensor - for i, name in enumerate(layer_names) - } - decode_connector.register_kv_caches(kv_caches) - - remote_config_params: dict[str, Any] = { - "model": "facebook/opt-125m", - "block_size": block_size, - **config_overrides, - } - remote_vllm_config = create_vllm_config(**remote_config_params) - - with contextlib.ExitStack() as stack: - if "vllm_version" in version_override: - stack.enter_context( - patch("vllm.__version__", version_override["vllm_version"]) - ) - elif "connector_version" in version_override: - stack.enter_context( - patch.object( - nixl.metadata, - "NIXL_CONNECTOR_VERSION", - version_override["connector_version"], - ) - ) - remote_hash = compute_nixl_compatibility_hash( - remote_vllm_config, - decode_worker.backend_name, - ) - - prefill_block_size = remote_vllm_config.cache_config.block_size - prefill_block_lens = [4096 * prefill_block_size] - prefill_metadata = NixlAgentMetadata( - engine_id=FakeNixlConnectorWorker.REMOTE_ENGINE_ID, - agent_metadata=FakeNixlWrapper.AGENT_METADATA, - kv_caches_base_addr=[0], - device_id=0, - num_blocks=1, - block_lens=prefill_block_lens, - block_strides=prefill_block_lens, - kv_cache_layout="LBHNC", - block_size=prefill_block_size, - ssm_sizes=(0, 0), - attn_backend_name=decode_worker.backend_name, - physical_blocks_per_logical_kv_block=1, - ) - handshake_payload = NixlHandshakePayload( - compatibility_hash=remote_hash, - agent_metadata_bytes=msgspec.msgpack.encode(prefill_metadata), - ) - - # Mock ZMQ socket to return our handshake payload - mock_socket = MagicMock() - mock_socket.recv_multipart.return_value = [ - msgspec.msgpack.encode(handshake_payload), - msgspec.msgpack.encode(time.perf_counter()), - ] - - # Mock add_remote_agent to avoid actual NIXL operations - # Patch zmq_ctx to return our mock socket - with ( - patch.object(decode_worker, "add_remote_agent", return_value="fake_agent"), - patch.object(nixl.base_worker, "zmq_ctx") as mock_zmq_ctx, - ): - mock_zmq_ctx.return_value.__enter__.return_value = mock_socket - - if should_fail: - with pytest.raises(RuntimeError, match="compatibility hash mismatch"): - decode_worker._nixl_handshake( - host="localhost", - port=1234, - remote_tp_size=1, - expected_engine_id=FakeNixlConnectorWorker.REMOTE_ENGINE_ID, - ) - else: - result, _ = decode_worker._nixl_handshake( - host="localhost", - port=1234, - remote_tp_size=1, - expected_engine_id=FakeNixlConnectorWorker.REMOTE_ENGINE_ID, - ) - # Verify handshake returned agent mapping - assert isinstance(result, dict) - assert len(result) == 1 - - -@pytest.mark.parametrize( - "error_scenario", - [ - "handshake_decode_error", - "handshake_validation_error", - "metadata_decode_error", - "metadata_validation_error", - ], -) -@patch( - "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper", - FakeNixlWrapper, -) -def test_handshake_decode_errors(default_vllm_config, dist_init, error_scenario): - """Test that msgspec decode errors are properly handled during handshake. - - Tests both DecodeError and ValidationError for both decoders: - - NixlHandshakePayload decoder - - NixlAgentMetadata decoder - """ - block_size = maybe_update_block_size(16) - local_vllm_config = create_vllm_config( - model="facebook/opt-125m", - block_size=block_size, - ) - decode_connector = NixlConnector( - local_vllm_config, - KVConnectorRole.WORKER, - make_kv_cache_config(block_size=block_size), - ) - decode_worker = decode_connector.connector_worker - - backend = get_current_attn_backend(local_vllm_config) - probe_spec = decode_worker.kv_cache_config.kv_cache_groups[0].kv_cache_spec - test_shape = compute_layer_kv_cache_shape_bytes(probe_spec, 1) - decode_worker.transfer_topo = TransferTopology( - tp_rank=decode_worker.tp_rank, - tp_size=decode_worker.world_size, - block_size=decode_worker.block_size, - engine_id=decode_worker.engine_id, - is_mla=decode_worker.use_mla, - is_mamba=False, - total_num_kv_heads=decode_worker.model_config.get_total_num_kv_heads(), - attn_backends=[backend], - tensor_shape=test_shape, - ) - - decode_worker.compat_hash = compute_nixl_compatibility_hash( - decode_worker.vllm_config, - decode_worker.backend_name, - ) - - if error_scenario == "handshake_decode_error": - msg_bytes = b"this is not valid msgpack data" - elif error_scenario == "handshake_validation_error": - msg_bytes = msgspec.msgpack.encode({"wrong_field": "value"}) - elif error_scenario == "metadata_decode_error": - valid_handshake = NixlHandshakePayload( - compatibility_hash=decode_worker.compat_hash, - agent_metadata_bytes=b"invalid msgpack for metadata", - ) - msg_bytes = msgspec.msgpack.encode(valid_handshake) - - elif error_scenario == "metadata_validation_error": - valid_handshake = NixlHandshakePayload( - compatibility_hash=decode_worker.compat_hash, - agent_metadata_bytes=msgspec.msgpack.encode({"missing": "fields"}), - ) - msg_bytes = msgspec.msgpack.encode(valid_handshake) - else: - raise AssertionError(f"{error_scenario} not a valid scenario") - - mock_socket = MagicMock() - mock_socket.recv_multipart.return_value = [ - msg_bytes, - msgspec.msgpack.encode(time.perf_counter()), - ] - with ( - patch.object(decode_worker, "add_remote_agent", return_value="fake_agent"), - patch.object(nixl.base_worker, "zmq_ctx") as mock_zmq_ctx, - ): - mock_zmq_ctx.return_value.__enter__.return_value = mock_socket - - with pytest.raises(RuntimeError): - decode_worker._nixl_handshake( - host="localhost", - port=1234, - remote_tp_size=1, - expected_engine_id=FakeNixlConnectorWorker.REMOTE_ENGINE_ID, - ) - - @patch( - "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper", - FakeNixlWrapper, - ) - def test_mla_broadcast_notif_uses_remote_request_id( - self, default_vllm_config, dist_init - ): - """MLA + remote TP > local TP: the broadcast notification sent to - non-read prefill ranks must be keyed by the prefill-side request - id (``meta.remote.request_id``), not the local decode request id. - - Prefill ranks key ``_reqs_to_send`` by their own request id, so a - broadcast keyed by the decode id is rejected in - ``_get_new_notifs`` with "Potentially invalid KV blocks for - unrecognized request" and the blocks only release via the abort - timeout. See ``_read_blocks_for_req`` in - ``vllm/distributed/kv_transfer/kv_connector/v1/nixl/worker.py``. - """ - decode_tp_size = 1 - prefill_tp_size = 4 - - block_size = maybe_update_block_size(16) - vllm_config = create_vllm_config(block_size=block_size) - vllm_config.parallel_config.tensor_parallel_size = decode_tp_size - - connector = NixlConnector( - vllm_config, - KVConnectorRole.WORKER, - make_kv_cache_config(block_size=block_size), - ) - connector.connector_worker = FakeNixlConnectorWorker( - vllm_config, connector.engine_id, hand_shake_latency=0 - ) - worker = connector.connector_worker - - # Force the MLA path; only `self.use_mla` gates the branches we - # exercise inside `_read_blocks_for_req`. - worker.use_mla = True - - # Manually register the remote (P) engine and pre-populate the - # per-rank state the handshake would normally fill in. The real - # `_nixl_handshake` is unnecessary here — we only need - # `transfer_topo` to know `remote_tp_size`, and `_remote_agents` - # / `dst_xfer_side_handles` to be keyed by remote rank. - remote_engine_id = "remote_engine" - worker.transfer_topo.register_remote_engine( - remote_engine_id=remote_engine_id, - remote_tp_size=prefill_tp_size, - remote_block_size=worker.block_size, - remote_block_len=worker.block_size * 4096, - remote_physical_blocks_per_logical=1, - local_block_len=worker.block_size * 4096, - ) - worker._remote_agents[remote_engine_id] = { - (0, rank): f"agent_p{rank}" for rank in range(prefill_tp_size) - } - worker.dst_xfer_side_handles = { - remote_engine_id: {rank: 100 + rank for rank in range(prefill_tp_size)} - } - # Sanity: D TP=1, P TP=4 => tp_ratio = -4 (P > D). - assert worker.transfer_topo.tp_ratio(prefill_tp_size) == -prefill_tp_size - - # Distinct ids on each side — that's the whole point of the bug. - decode_req_id = "decode-req-AAAA" - prefill_req_id = "prefill-req-BBBB" - assert decode_req_id != prefill_req_id - - metadata = NixlConnectorMetadata() - metadata.add_new_req_to_recv( - request_id=decode_req_id, - local_block_ids=([0, 1, 2],), - kv_transfer_params={ - "remote_block_ids": ([10, 11, 12],), - "remote_engine_id": remote_engine_id, - "remote_request_id": prefill_req_id, - "remote_host": "localhost", - "remote_port": 1234, - "remote_tp_size": prefill_tp_size, - }, - ) - meta = metadata.reqs_to_recv[decode_req_id] - - # Capture broadcast send_notif calls; stub `_read_blocks` so we - # don't need a working xfer path. Real `_read_blocks` emits its - # auto-notif via `make_prepped_xfer`, not via `send_notif`, so - # any captured `send_notif` here is a broadcast. - send_notif_calls: list[tuple[str, bytes]] = [] - worker.nixl_wrapper.send_notif = ( # type: ignore[method-assign] - lambda agent_name, notif_msg: send_notif_calls.append( - (agent_name, notif_msg) - ) - ) - worker._read_blocks = MagicMock() # type: ignore[method-assign] - - worker._read_blocks_for_req(decode_req_id, meta) - - # MLA: read once from rank 0 and broadcast to the other ranks. - worker._read_blocks.assert_called_once() - assert worker._read_blocks.call_args.kwargs["remote_rank"] == 0 - assert ( - worker._read_blocks.call_args.kwargs["remote_request_id"] == prefill_req_id - ) - - # Broadcast goes to ranks {1, 2, 3} only, never to the read target. - expected_recipients = { - worker._remote_agents[remote_engine_id][(0, r)] - for r in range(1, prefill_tp_size) - } - assert {agent for agent, _ in send_notif_calls} == expected_recipients - - # Every broadcast notif must be keyed by the prefill request id. - # Pre-fix this used the *decode* request id, which prefill ranks - # didn't recognize. - expected_notif = f"{prefill_req_id}:{decode_tp_size}".encode() - bad_notif = f"{decode_req_id}:{decode_tp_size}".encode() - for agent, notif in send_notif_calls: - assert notif == expected_notif, ( - f"Broadcast notif to {agent!r} must use prefill_req_id; " - f"got {notif!r} (expected {expected_notif!r}, " - f"buggy form would be {bad_notif!r})" - ) - - -@patch( - "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper", - FakeNixlWrapper, -) -def test_kv_both_deprecation_warning(default_vllm_config, dist_init): - """kv_role='kv_both' should emit a deprecation log warning.""" - from vllm.logger import _print_warning_once - - _print_warning_once.cache_clear() - - block_size = maybe_update_block_size(16) - vllm_config = create_vllm_config(kv_role="kv_both", block_size=block_size) - - with patch( - "vllm.distributed.kv_transfer.kv_connector.v1.nixl.connector.logger" - ) as mock_logger: - mock_logger.warning_once = mock_logger.warning_once - NixlConnector( - vllm_config, - KVConnectorRole.WORKER, - make_kv_cache_config(block_size=block_size), - ) - - mock_logger.warning_once.assert_called_once() - msg = mock_logger.warning_once.call_args[0][0] - assert "kv_role='kv_both'" in msg - assert "deprecated" in msg - - -@patch( - "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper", - FakeNixlWrapper, -) -def test_explicit_kv_role_no_deprecation_warning(default_vllm_config, dist_init): - """kv_role='kv_consumer' or 'kv_producer' should NOT emit a warning.""" - for role in ("kv_consumer", "kv_producer"): - block_size = maybe_update_block_size(16) - vllm_config = create_vllm_config(kv_role=role, block_size=block_size) - with patch( - "vllm.distributed.kv_transfer.kv_connector.v1.nixl.connector.logger" - ) as mock_logger: - NixlConnector( - vllm_config, - KVConnectorRole.WORKER, - make_kv_cache_config(block_size=block_size), - ) - - ( - mock_logger.warning_once.assert_not_called(), - (f"kv_role={role!r} should not emit deprecation warning"), - ) diff --git a/tests/v1/kv_connector/unit/test_nixl_heartbeat.py b/tests/v1/kv_connector/unit/test_nixl_heartbeat.py deleted file mode 100644 index f2d95daa48..0000000000 --- a/tests/v1/kv_connector/unit/test_nixl_heartbeat.py +++ /dev/null @@ -1,253 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Unit tests for the scheduler-driven heartbeat / lease-renewal system.""" - -import time -from unittest.mock import MagicMock - -import pytest - -from vllm.v1.outputs import KVConnectorOutput - -from .utils import create_request, make_nixl_scheduler - -_ENGINE_A = "my-engine-id" - - -def _sched(kv_lease_duration: int = 30): - return make_nixl_scheduler(heartbeat=True, kv_lease_duration=kv_lease_duration) - - -def _req(request_id: int = 1): - return create_request(request_id=request_id, do_remote_prefill=True) - - -def _worker_stub(): - from vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker import ( - NixlConnectorWorker, - ) - - w = object.__new__(NixlConnectorWorker) - w._reqs_to_send = {} - w._lease_extension = 20 - w._reqs_to_process = set() - w.consumer_notification_counts_by_req = {} - w.expected_consumer_notifications_by_req = {} - w.xfer_stats = MagicMock() - return w - - -# =================================================================== -# Scheduler: on_new_request -# =================================================================== - - -def test_on_new_request_tracks_and_groups(): - """Add two reqs to same engine, one to another; verify grouping.""" - s = _sched() - s.on_new_request(_req(1)) - s.on_new_request(_req(2)) - - assert s._heartbeat_by_engine[_ENGINE_A].req_ids == {"prefill-1", "prefill-2"} - info = s._heartbeat_by_engine[_ENGINE_A] - assert (info.host, info.port, info.tp_size) == ("my-host", 1234, 1) - assert s._heartbeat_req_engine["id-1"] == (_ENGINE_A, "prefill-1") - - # Different engine. - r3 = _req(3) - r3.kv_transfer_params["remote_engine_id"] = "engine-b" - s.on_new_request(r3) - assert len(s._heartbeat_by_engine) == 2 - - -@pytest.mark.parametrize( - "make_req", - [ - lambda: create_request(request_id=2, do_remote_decode=True), - lambda: create_request(request_id=3), # no kv_transfer_params - ], - ids=["decode", "plain"], -) -def test_on_new_request_ignores_non_prefill(make_req): - s = _sched() - s.on_new_request(make_req()) - assert len(s._heartbeat_by_engine) == 0 - - -# =================================================================== -# Scheduler: _stop_heartbeat -# =================================================================== - - -def test_stop_heartbeat_partial_and_full(): - """Stop one of two reqs on same engine, then stop the other.""" - s = _sched() - s.on_new_request(_req(1)) - s.on_new_request(_req(2)) - - s._stop_heartbeat("id-1") - assert s._heartbeat_by_engine[_ENGINE_A].req_ids == {"prefill-2"} - assert "id-1" not in s._heartbeat_req_engine - - s._stop_heartbeat("id-2") - assert len(s._heartbeat_by_engine) == 0 - assert len(s._heartbeat_req_engine) == 0 - - -# =================================================================== -# Scheduler: build_connector_meta throttling -# =================================================================== - - -def test_build_connector_meta_heartbeat_throttling(): - # kv_lease_duration=30 => _heartbeat_interval = 30 // 6 = 5 - s = _sched(kv_lease_duration=30) - s.on_new_request(_req(1)) - - # Ensure the first call triggers by placing last_heartbeat far in the past. - s._last_heartbeat_time = time.perf_counter() - 10 - meta1 = s.build_connector_meta(MagicMock()) - assert _ENGINE_A in meta1.heartbeat_by_engine - - # Immediate second call is throttled (< 5s since last). - meta2 = s.build_connector_meta(MagicMock()) - assert len(meta2.heartbeat_by_engine) == 0 - - -# =================================================================== -# Scheduler: cleanup paths (update_connector_output / request_finished) -# =================================================================== - - -def test_update_connector_output_stops_heartbeat(): - s = _sched() - s.on_new_request(_req(1)) - - s.update_connector_output( - KVConnectorOutput( - finished_sending=None, - finished_recving={"id-1"}, - invalid_block_ids=set(), - ) - ) - - assert len(s._heartbeat_by_engine) == 0 - assert len(s._heartbeat_req_engine) == 0 - - -def test_request_finished_stops_heartbeat(): - s = _sched() - r = _req(1) - s.on_new_request(r) - - # Simulate update_state_after_alloc having consumed do_remote_prefill. - r.kv_transfer_params["do_remote_prefill"] = False - s.request_finished(r, block_ids=()) - - assert len(s._heartbeat_by_engine) == 0 - assert len(s._heartbeat_req_engine) == 0 - - -# =================================================================== -# Worker: _handle_heartbeat -# =================================================================== - - -def test_handle_heartbeat(): - w = _worker_stub() - far_future = time.perf_counter() + 99999 - w._reqs_to_send = {"req-a": 100.0, "req-b": far_future} - - before = time.perf_counter() - w._handle_heartbeat("req-a,req-b,req-unknown") - - # req-a: pushed forward to ~now+20. - assert w._reqs_to_send["req-a"] >= before + 20 - # req-b: already far out, max() keeps it. - assert w._reqs_to_send["req-b"] >= far_future - # req-unknown: not added. - assert "req-unknown" not in w._reqs_to_send - - -def test_reaper_reclaims_expired_leases_behind_heartbeated_head(monkeypatch): - """A heartbeated head entry must not strand expired leases behind it. - - Mirrors the stdlib repro in https://github.com/vllm-project/vllm/issues/58222: - A is inserted first and kept alive via heartbeats; B and C expire and - must still be reaped. The old early-exit scan returned nothing here. - """ - w = _worker_stub() - w._reqs_to_process.update(("A", "B", "C")) - # Insertion order is A, B, C. Heartbeats will keep A at the head. - w._reqs_to_send = {"A": 30.0, "B": 32.0, "C": 34.0} - - from vllm.distributed.kv_transfer.kv_connector.v1.nixl import base_worker - - now = {"t": 0.0} - monkeypatch.setattr(base_worker.time, "perf_counter", lambda: now["t"]) - for t in (20.0, 40.0, 60.0, 80.0): - now["t"] = t - w._handle_heartbeat("A") - - # A was renewed in place: still the dict head, expiry 100. - assert next(iter(w._reqs_to_send)) == "A" - assert w._reqs_to_send["A"] == pytest.approx(100.0) - - done: set[str] = set() - now["t"] = 50.0 - w._reap_expired_send_leases(done) - - assert done == {"B", "C"} - assert list(w._reqs_to_send) == ["A"] - assert w._reqs_to_process == {"A"} - assert w.xfer_stats.record_kv_expired_req.call_count == 2 - - # now == expires is expired (now >= expires): A is due at 100. - done_eq: set[str] = set() - now["t"] = 100.0 - w._reap_expired_send_leases(done_eq) - assert done_eq == {"A"} - assert not w._reqs_to_send - assert not w._reqs_to_process - assert w.xfer_stats.record_kv_expired_req.call_count == 3 - - -def test_reaper_reclaims_when_now_equals_expiry(monkeypatch): - """The reaper must reclaim on the now == expires boundary.""" - w = _worker_stub() - w._reqs_to_process.add("eq") - w._reqs_to_send = {"eq": 50.0} - - from vllm.distributed.kv_transfer.kv_connector.v1.nixl import base_worker - - monkeypatch.setattr(base_worker.time, "perf_counter", lambda: 50.0) - - done: set[str] = set() - w._reap_expired_send_leases(done) - - assert done == {"eq"} - assert not w._reqs_to_send - assert not w._reqs_to_process - w.xfer_stats.record_kv_expired_req.assert_called_once() - - -def test_reaper_reclaims_shorter_lease_behind_later_deadline(monkeypatch): - """A later-inserted longer TTL must not block an earlier-expiring entry. - - Delete+reinsert on heartbeat cannot restore expiry order in this case - (issue #58222), so the reaper must scan the whole map. - """ - w = _worker_stub() - w._reqs_to_process.update(("X", "Y")) - w._reqs_to_send = {"X": 55.0, "Y": 46.0} - - from vllm.distributed.kv_transfer.kv_connector.v1.nixl import base_worker - - monkeypatch.setattr(base_worker.time, "perf_counter", lambda: 50.0) - - done: set[str] = set() - w._reap_expired_send_leases(done) - - assert done == {"Y"} - assert list(w._reqs_to_send) == ["X"] - assert w._reqs_to_process == {"X"} diff --git a/tests/v1/kv_connector/unit/test_nixl_push_connector.py b/tests/v1/kv_connector/unit/test_nixl_push_connector.py deleted file mode 100644 index e2f9e99b49..0000000000 --- a/tests/v1/kv_connector/unit/test_nixl_push_connector.py +++ /dev/null @@ -1,2156 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Unit tests for NixlPushConnector (scheduler + worker). - -These tests cover the end-to-end mechanics of the push design without -requiring a real NIXL agent or network: - -* Scheduler stages D registrations on ``update_state_after_alloc`` and - P finished blocks on ``request_finished``. -* ``build_connector_meta`` drains them onto - ``meta.push_registrations`` / ``meta.push_finished_blocks``. -* ``has_pending_push_work`` reports True/False over the lifecycle. -* ``update_connector_output`` clears state on ``finished_sending`` and - ``finished_recving``. -* The worker matches D registrations against P finished blocks (both - scenario directions) and forwards non-PUSH_REG NIXL notifs to the main - thread's ``_get_new_notifs``. -* ``get_transfer_results`` enqueues evictions for the writer. -""" - -from __future__ import annotations - -import logging -import queue -import threading -import time -from collections import defaultdict -from concurrent.futures import Future -from typing import Any -from unittest.mock import MagicMock, patch - -import msgspec -import pytest - -from vllm.distributed.kv_transfer.kv_connector.utils import TransferTopology -from vllm.distributed.kv_transfer.kv_connector.v1.base import ( - KVConnectorTransferResults, -) -from vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker import ( - NixlBaseConnectorWorker, -) -from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import ( - PUSH_REG_NOTIF_PREFIX, - NixlAgentMetadata, - NixlConnectorMetadata, -) -from vllm.distributed.kv_transfer.kv_connector.v1.nixl.push_worker import ( - NixlPushConnectorWorker, -) -from vllm.distributed.kv_transfer.kv_connector.v1.nixl.stats import ( - NixlKVConnectorStats, -) -from vllm.distributed.kv_transfer.kv_connector.v1.nixl.tp_mapping import ( - ReadSpec, - TPMapping, -) -from vllm.distributed.kv_transfer.kv_connector.v1.nixl.utils import ( - get_base_request_id, -) -from vllm.v1.kv_cache_interface import FullAttentionSpec -from vllm.v1.outputs import KVConnectorOutput - -from .utils import create_request, expand_strided_descs, make_nixl_push_scheduler - -# ----------------------------------------------------------------- # -# Helpers / fakes # -# ----------------------------------------------------------------- # - - -def _make_request( - *, - request_id: str, - is_d_side: bool = True, - remote_engine_id: str = "prefill-engine", - remote_request_id: str | None = None, - remote_host: str = "10.0.0.1", - remote_port: int = 5601, - tp_size: int = 1, - finished: bool = True, -) -> MagicMock: - """Build a minimal Request mock used by request_finished.""" - from vllm.v1.request import RequestStatus - - req = MagicMock() - req.request_id = request_id - req.num_computed_tokens = 64 - - if is_d_side: - # D-side request: do_remote_prefill=True -> prefill on a remote P. - params: dict[str, Any] = { - "do_remote_prefill": True, - "do_remote_decode": False, - "remote_engine_id": remote_engine_id, - "remote_request_id": remote_request_id or f"prefill-{request_id}", - "remote_host": remote_host, - "remote_port": remote_port, - "tp_size": tp_size, - } - else: - # P-side request: do_remote_decode=True (we are the prefiller). - params = { - "do_remote_prefill": False, - "do_remote_decode": True, - } - req.kv_transfer_params = params - req.status = ( - RequestStatus.FINISHED_LENGTH_CAPPED if finished else RequestStatus.RUNNING - ) - return req - - -class _BlocksMock: - """Minimal stand-in for ``KVCacheBlocks`` used in update_state_after_alloc.""" - - def __init__(self, block_ids: tuple[list[int], ...]): - self._block_ids = block_ids - - def get_unhashed_block_ids_all_groups(self) -> tuple[list[int], ...]: - return self._block_ids - - -def _stub_sw_clipping(scheduler) -> None: - """Make ``get_exchange_clipped_blocks`` a passthrough so tests don't - need the full sliding-window machinery.""" - scheduler.get_exchange_clipped_blocks = lambda block_ids, clip_ssm=True: block_ids - - -# ----------------------------------------------------------------- # -# Scheduler-side tests # -# ----------------------------------------------------------------- # - - -class TestPushScheduler: - def test_p_side_mamba_truncates_before_cache_lookup(self): - """Push mode normalizes P-side Mamba requests before cache lookup.""" - sched = make_nixl_push_scheduler(has_mamba=True) - request = create_request(num_tokens=10, do_remote_decode=True) - original_len = len(request.prompt_token_ids) - - sched.on_new_request(request) - - assert len(request.prompt_token_ids) == original_len - 1 - assert request.kv_transfer_params["_p_side_truncated"] is True - - with patch.object( - sched, - "_truncate_request_for_prefill", - side_effect=AssertionError("must not truncate after cache lookup"), - ): - assert sched.get_num_new_matched_tokens(request, 0) == (0, False) - - def test_d_side_update_state_after_alloc_stages_registration(self): - """D scheduler stashes registration data + arms watchdog deadline.""" - sched = make_nixl_push_scheduler() - _stub_sw_clipping(sched) - - request = _make_request(request_id="req-d-1") - blocks = _BlocksMock(block_ids=([10, 11, 12],)) - - sched.update_state_after_alloc(request, blocks, num_external_tokens=48) - - assert request.request_id in sched._push_pending_registrations - reg = sched._push_pending_registrations[request.request_id] - # ``request_id`` is D's own vLLM request id; plus our own (D) coords. - assert reg["request_id"] == request.request_id - assert reg["decode_engine_id"] == sched.engine_id - assert reg["decode_host"] == sched.side_channel_host - assert reg["decode_port"] == sched.side_channel_port - assert reg["local_block_ids"] == ([10, 11, 12],) - assert reg["remote_engine_id"] == "prefill-engine" - - # Watchdog deadline set in the future. - deadline = sched._push_registration_deadlines[request.request_id] - assert deadline > time.perf_counter() - # do_remote_prefill flipped off so the request isn't reprocessed. - assert request.kv_transfer_params["do_remote_prefill"] is False - # Tracked as awaiting a recv. - assert request.request_id in sched._reqs_need_recv - - def test_p_side_request_finished_stages_blocks(self): - """P scheduler pushes blocks into both _finished_request_blocks (lease) - and _newly_finished_push_blocks (metadata for next step).""" - sched = make_nixl_push_scheduler() - _stub_sw_clipping(sched) - - request = _make_request(request_id="req-p-1", is_d_side=False) - block_ids = ([20, 21, 22, 23],) - - delay, ret_params = sched.request_finished(request, block_ids) - - assert delay is True - assert ret_params is not None - assert ret_params["do_remote_prefill"] is True - assert ret_params["do_remote_decode"] is False - assert request.request_id in sched._finished_request_blocks - assert request.request_id in sched._newly_finished_push_blocks - assert request.request_id in sched._reqs_need_send # lease armed - - def test_build_connector_meta_drains_both_sides(self): - """meta.push_registrations and meta.push_finished_blocks are filled - from the staging dicts and the staging dicts are cleared.""" - sched = make_nixl_push_scheduler() - _stub_sw_clipping(sched) - - # Stage one D registration and one P finished entry. - d_req = _make_request(request_id="req-d-9") - sched.update_state_after_alloc( - d_req, _BlocksMock(([1, 2, 3],)), num_external_tokens=48 - ) - p_req = _make_request(request_id="req-p-9", is_d_side=False) - sched.request_finished(p_req, ([4, 5, 6],)) - - scheduler_output = MagicMock() - scheduler_output.scheduled_new_reqs = [] - scheduler_output.scheduled_cached_reqs = MagicMock( - req_ids=[], resumed_req_ids=set() - ) - - # Patch parent build_connector_meta so we don't have to set up - # all the base scheduler plumbing. - with patch.object( - sched.__class__.__mro__[1], - "build_connector_meta", - return_value=NixlConnectorMetadata(), - ): - meta = sched.build_connector_meta(scheduler_output) - - assert isinstance(meta, NixlConnectorMetadata) - assert "req-d-9" in meta.push_registrations - assert "req-p-9" in meta.push_finished_blocks - # Staging dicts cleared. - assert sched._push_pending_registrations == {} - assert sched._newly_finished_push_blocks == {} - # Lease bookkeeping kept until the WRITE completes. - assert "req-p-9" in sched._finished_request_blocks - - def test_has_pending_push_work_lifecycle(self): - sched = make_nixl_push_scheduler() - _stub_sw_clipping(sched) - - assert sched.has_pending_push_work() is False - - # P finished blocks waiting for WRITE completion. - p_req = _make_request(request_id="req-p-7", is_d_side=False) - sched.request_finished(p_req, ([0, 1],)) - assert sched.has_pending_push_work() is True - - # Drain via build_connector_meta - lease still pending until WRITE. - scheduler_output = MagicMock() - scheduler_output.scheduled_new_reqs = [] - scheduler_output.scheduled_cached_reqs = MagicMock( - req_ids=[], resumed_req_ids=set() - ) - with patch.object( - sched.__class__.__mro__[1], - "build_connector_meta", - return_value=NixlConnectorMetadata(), - ): - sched.build_connector_meta(scheduler_output) - # Lease is pending until WRITE completes -> still True. - assert sched.has_pending_push_work() is True - - # Simulate WRITE completion via update_connector_output. - sched.update_connector_output( - KVConnectorOutput( - finished_sending={"req-p-7"}, - finished_recving=set(), - invalid_block_ids=set(), - ) - ) - assert sched.has_pending_push_work() is False - - def test_update_connector_output_clears_lease_and_watchdog(self): - sched = make_nixl_push_scheduler() - _stub_sw_clipping(sched) - - d_req = _make_request(request_id="req-d-x") - sched.update_state_after_alloc( - d_req, _BlocksMock(([1, 2],)), num_external_tokens=32 - ) - p_req = _make_request(request_id="req-p-x", is_d_side=False) - sched.request_finished(p_req, ([3, 4],)) - - sched.update_connector_output( - KVConnectorOutput( - finished_sending={"req-p-x"}, - finished_recving={"req-d-x"}, - invalid_block_ids=set(), - ) - ) - assert "req-p-x" not in sched._finished_request_blocks - assert "req-d-x" not in sched._push_registration_deadlines - - def test_registration_watchdog_expires(self, caplog): - """Stale D registrations whose deadline has passed are dropped at - ``build_connector_meta`` time.""" - # Watchdog logs a WARNING when it drops the stale entry; that's - # what this test is verifying, so silence it in the test report. - caplog.set_level( - logging.CRITICAL, - logger=("vllm.distributed.kv_transfer.kv_connector.v1.nixl.push_scheduler"), - ) - sched = make_nixl_push_scheduler() - _stub_sw_clipping(sched) - - d_req = _make_request(request_id="req-d-stale") - sched.update_state_after_alloc( - d_req, _BlocksMock(([7, 8],)), num_external_tokens=32 - ) - # Force the deadline into the past. - sched._push_registration_deadlines[d_req.request_id] = time.perf_counter() - 1.0 - - scheduler_output = MagicMock() - scheduler_output.scheduled_new_reqs = [] - scheduler_output.scheduled_cached_reqs = MagicMock( - req_ids=[], resumed_req_ids=set() - ) - with patch.object( - sched.__class__.__mro__[1], - "build_connector_meta", - return_value=NixlConnectorMetadata(), - ): - meta = sched.build_connector_meta(scheduler_output) - - assert d_req.request_id not in sched._push_registration_deadlines - assert d_req.request_id not in sched._push_pending_registrations - assert d_req.request_id not in meta.push_registrations - - -# ----------------------------------------------------------------- # -# Worker-side tests # -# ----------------------------------------------------------------- # - - -class _StubWriterWorker(NixlPushConnectorWorker): - """Construct a worker without invoking ``__init__`` so we can drive - the matching/notif logic without bringing up NIXL or torch.""" - - @classmethod - def fresh(cls) -> _StubWriterWorker: - w = object.__new__(cls) - - # Push-specific state managed by NixlPushConnectorWorker. - from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import ( - ReqId, - TransferHandle, - ) - - w._sending_transfers = defaultdict[ReqId, list[TransferHandle]](list) - w._send_failures = set() - w._sending_transfers_lock = threading.Lock() - w._push_finished_blocks = {} - w._pending_d_registrations = {} - w._reg_send_inbox = queue.Queue() - w._finished_blocks_inbox = queue.Queue() - w._pending_completion_notifs = queue.Queue() - w._evict_finished_inbox = queue.Queue() - w._deferred_push_inbox = queue.Queue() - w._push_writer_wake = threading.Event() - w._push_writer_stop = threading.Event() - w._push_writer_thread = None - - # Base worker fields touched by start_load_kv / _get_new_notifs. - w._recving_metadata = {} - w._pending_recv_notifs = {} - w._failed_recv_reqs = queue.Queue() - w._recv_failures = set() - w._recving_transfers = defaultdict(list) - w._is_hma_required = False - w.xfer_stats = NixlKVConnectorStats() - w._has_packed_cache = False - w._reqs_to_process = set() - w._reqs_to_send = {} - w.consumer_notification_counts_by_req = defaultdict(int) - w.tp_rank = 0 - w.pcp_rank = 0 - w.pcp_dcp_sharded = False - w.world_size = 1 - w.pp_size = 1 - w.engine_id = "test-decode-engine" - w._remote_agents = {} - w._failed_remote_engines = set() - w._invalid_remote_engines = set() - w._handshake_lock = threading.RLock() - w._physical_blocks_per_logical_kv_block = 1 - w._uses_region_group_mapping = False - w.region_group_ids = [0] - w._transfer_layer_names = () - w._transfer_layer_region_indices = () - w._packed_member_layouts = {} - w._transfer_layer_group_ids = () - w.dst_region_num_blocks = {} - w.dst_region_group_ids = {} - w.dst_uses_region_group_mapping = {} - w.dst_region_mem_types = {} - # Single non-hybrid attention group, matching the stub block id lists. - w._has_mamba = False - w._is_csa_linear = False - w._group_spec_types = (FullAttentionSpec,) - w._engine_ttl = 0.0 - w._engine_last_active = {} - w._engine_by_address = {} - - # Track _do_start_push_kv invocations. - calls: list[tuple[str, Any, dict[str, Any]]] = [] - w.start_push_calls = calls - return w - - def _do_start_push_kv( - self, - request_id: str, - local_block_ids, - registration_data: dict[str, Any], - ) -> None: # pragma: no cover - exercised through tests - # Track the call instead of issuing real WRITEs. - self.start_push_calls.append((request_id, local_block_ids, registration_data)) - - -def _registration_data( - request_id: str, - *, - decode_engine_id: str = "decode-engine", - decode_host: str = "10.0.0.2", - decode_port: int = 5602, - decode_tp_size: int = 1, - local_block_ids=((100, 101, 102),), - remote_engine_id: str = "prefill-engine", - remote_host: str = "10.0.0.1", - remote_port: int = 5601, - remote_tp_size: int = 1, -) -> dict[str, Any]: - return { - "request_id": request_id, - "decode_engine_id": decode_engine_id, - "decode_host": decode_host, - "decode_port": decode_port, - "decode_tp_size": decode_tp_size, - "local_block_ids": local_block_ids, - "remote_engine_id": remote_engine_id, - "remote_host": remote_host, - "remote_port": remote_port, - "remote_tp_size": remote_tp_size, - } - - -class TestPushWriterRegSend: - """``_do_send_reg_notif``: the PUSH_REG goes to every handshaked agent of - the P engine, and the agent map is read under ``_handshake_lock`` so the - send can't race the handshake done-callback's insert or a concurrent - stale-engine eviction (X1).""" - - def _reg_send_worker(self) -> tuple[_StubWriterWorker, list]: - w = _StubWriterWorker.fresh() - w.nixl_wrapper = MagicMock() - failures: list = [] - w._handle_failed_transfer = lambda *a: failures.append(a) - return w, failures - - def test_sends_to_all_agents_with_prefix_and_payload(self): - w, failures = self._reg_send_worker() - w._remote_agents["prefill-engine"] = {(0, 0): "agent-0", (0, 1): "agent-1"} - rd = _registration_data("req-1") - - w._do_send_reg_notif("req-1", rd) - - assert failures == [] - calls = w.nixl_wrapper.send_notif.call_args_list - assert len(calls) == 2 - assert {c.args[0] for c in calls} == {"agent-0", "agent-1"} - for call in calls: - notif = call.kwargs["notif_msg"] - assert notif.startswith(PUSH_REG_NOTIF_PREFIX) - # (Tuples decode back as lists, so compare identity fields.) - decoded = msgspec.msgpack.decode(notif[len(PUSH_REG_NOTIF_PREFIX) :]) - assert decoded["request_id"] == "req-1" - assert decoded["decode_engine_id"] == rd["decode_engine_id"] - assert decoded["remote_engine_id"] == rd["remote_engine_id"] - assert decoded["local_block_ids"] == [ - list(group) for group in rd["local_block_ids"] - ] - - def test_queues_failed_request_when_engine_not_handshaked(self): - w, failures = self._reg_send_worker() - - w._do_send_reg_notif("req-1", _registration_data("req-1")) - - assert failures == [] - assert w._failed_recv_reqs.get_nowait() == "req-1" - assert w._failed_recv_reqs.empty() - w.nixl_wrapper.send_notif.assert_not_called() - - def test_agent_read_is_serialized_with_handshake_lock(self): - """While another thread holds ``_handshake_lock`` (as the handshake - done-callback does while writing ``_remote_agents``), the writer's - check-then-send must not run ahead of the critical section.""" - w, failures = self._reg_send_worker() - w._remote_agents["prefill-engine"] = {(0, 0): "agent-0"} - rd = _registration_data("req-1") - - w._handshake_lock.acquire() - done = threading.Event() - errors: list[BaseException] = [] - - def _send(): - try: - w._do_send_reg_notif("req-1", rd) - except BaseException as e: # pragma: no cover - errors.append(e) - finally: - done.set() - - t = threading.Thread(target=_send) - t.start() - assert not done.wait(timeout=0.2) - w.nixl_wrapper.send_notif.assert_not_called() - assert failures == [] - - w._handshake_lock.release() - assert done.wait(timeout=2.0) - t.join(timeout=2.0) - assert errors == [] - assert failures == [] - assert w.nixl_wrapper.send_notif.call_count == 1 - - -class TestPushWriterMatching: - def test_handle_push_reg_matches_existing_finished_blocks(self): - """PUSH_REG arrives second (P finished first): match + fire.""" - w = _StubWriterWorker.fresh() - # P had already finished; its blocks were stashed via metadata. - w._push_finished_blocks["req-A"] = ([200, 201, 202],) - - notif = PUSH_REG_NOTIF_PREFIX + msgspec.msgpack.encode( - _registration_data("req-A") - ) - w._handle_push_reg_notif(notif) - - assert len(w.start_push_calls) == 1 - rid, blocks, reg = w.start_push_calls[0] - assert rid == "req-A" - assert blocks == ([200, 201, 202],) - assert reg["decode_engine_id"] == "decode-engine" - # Finished blocks consumed. - assert "req-A" not in w._push_finished_blocks - assert w._pending_d_registrations == {} - - def test_handle_push_reg_stashes_when_no_finished_blocks_yet(self): - """PUSH_REG arrives first (D registered first): stash, no fire.""" - w = _StubWriterWorker.fresh() - - notif = PUSH_REG_NOTIF_PREFIX + msgspec.msgpack.encode( - _registration_data("req-B") - ) - w._handle_push_reg_notif(notif) - - assert len(w.start_push_calls) == 0 - assert "req-B" in w._pending_d_registrations - - def test_handle_push_reg_matches_after_stripping_random_suffix(self): - """P and D assign the same logical request the same - ``cmpl--`` but different per-engine random suffixes; - the writer should still match P's finished blocks via the - suffix-stripping fallback in ``_pop_matching_finished_blocks``. - """ - w = _StubWriterWorker.fresh() - # Same base id + completion index; differ only in the trailing - # ``-<8 hex>`` randomization suffix. - p_id = "cmpl-12345678-aaaa-bbbb-cccc-1234567890ab-0-aaaaaaaa" - d_id = "cmpl-12345678-aaaa-bbbb-cccc-1234567890ab-0-bbbbbbbb" - # Sanity: same base id under the helper used by the connector. - assert get_base_request_id(p_id) == get_base_request_id(d_id) - - w._push_finished_blocks[p_id] = ([1, 2, 3],) - notif = PUSH_REG_NOTIF_PREFIX + msgspec.msgpack.encode(_registration_data(d_id)) - w._handle_push_reg_notif(notif) - - # Suffix-stripped fallback matched and fired. - assert len(w.start_push_calls) == 1 - assert w.start_push_calls[0][0] == p_id - assert p_id not in w._push_finished_blocks - - def test_handle_push_reg_drops_malformed(self, caplog): - # The writer logs WARNING/ERROR when it sees these bad payloads; - # that's the desired behavior, so suppress the noise from test - # output rather than letting it look like a failure. - caplog.set_level( - logging.CRITICAL, - logger=("vllm.distributed.kv_transfer.kv_connector.v1.nixl.push_worker"), - ) - w = _StubWriterWorker.fresh() - # Missing request_id -> should drop without raising. - bad = PUSH_REG_NOTIF_PREFIX + msgspec.msgpack.encode({"decode_engine_id": "x"}) - w._handle_push_reg_notif(bad) - assert w._pending_d_registrations == {} - assert w.start_push_calls == [] - - # Undecodable payload also dropped. - w._handle_push_reg_notif(PUSH_REG_NOTIF_PREFIX + b"\xff\xff\xff") - assert w.start_push_calls == [] - - -class TestPushWriterStartLoadKv: - def test_finished_blocks_inbox_matches_stashed_registration(self): - """Run the writer-loop's finished-blocks drain against a - pre-populated _pending_d_registrations entry.""" - w = _StubWriterWorker.fresh() - w._pending_d_registrations["req-C"] = _registration_data("req-C") - - # Simulate start_load_kv enqueuing finished blocks. - w._finished_blocks_inbox.put(("req-C", ([10, 11, 12],))) - - # Drain like the writer loop does. - while True: - try: - rid, blocks = w._finished_blocks_inbox.get_nowait() - except queue.Empty: - break - matched = w._pop_matching_registration(rid) - if matched is not None: - w._do_start_push_kv(rid, blocks, matched) - else: - w._push_finished_blocks[rid] = blocks - - assert len(w.start_push_calls) == 1 - assert w.start_push_calls[0][0] == "req-C" - assert "req-C" not in w._pending_d_registrations - - def test_start_load_kv_enqueues_to_writer(self): - """``start_load_kv`` should hand registrations + finished blocks - to the writer queues without doing matching itself.""" - w = _StubWriterWorker.fresh() - # Stub heartbeats to a no-op; tests don't exercise the heartbeat - # path here. - w._send_heartbeats = lambda metadata: None - # Stub logical-to-kernel mapping used by reqs_to_recv. - w._logical_to_kernel_block_ids = lambda x, ratio: x - - meta = NixlConnectorMetadata() - meta.push_registrations = { - "req-D": _registration_data("req-D"), - } - meta.push_finished_blocks = { - "req-E": ([5, 6, 7],), - } - - w.start_load_kv(meta) - - # Things are queued for the writer; nothing fires yet. - assert w._reg_send_inbox.qsize() == 1 - assert w._finished_blocks_inbox.qsize() == 1 - assert w._push_writer_wake.is_set() - assert w.start_push_calls == [] - - @pytest.mark.parametrize("sharded", [False, True]) - def test_noncanonical_pcp_rank_only_pushes_distinct_shards(self, sharded): - w = _StubWriterWorker.fresh() - w.pcp_rank = 1 - w.pcp_dcp_sharded = sharded - w._send_heartbeats = MagicMock() - - meta = NixlConnectorMetadata() - meta.push_finished_blocks["req"] = ([1, 2],) - meta.reqs_in_batch.add("req") - meta.reqs_to_send["req"] = time.perf_counter() + 10 - - w.start_load_kv(meta) - - assert w._finished_blocks_inbox.empty() is (not sharded) - assert ("req" in w._reqs_to_process) is sharded - assert ("req" in w._reqs_to_send) is sharded - assert w._push_writer_wake.is_set() is sharded - assert w._send_heartbeats.called is sharded - - -# The P→D handshake must run on the base worker's background executor, never -# blocking the writer thread: ``_do_start_push_kv`` defers the WRITE until the -# handshake resolves, then re-drives via ``_deferred_push_inbox``. These call -# the *real* ``_do_start_push_kv`` (the stub overrides it for matching tests). -def _real_do_start_push_kv(w, *args): - return NixlPushConnectorWorker._do_start_push_kv(w, *args) - - -def test_do_start_push_kv_defers_then_writes_when_handshake_ready(): - """Full happy-path lifecycle: an in-flight handshake defers the WRITE (no - NIXL op from the writer or the executor callback); once it resolves the - request is re-queued on ``_deferred_push_inbox`` with the wake set; and on - re-drive with the handshake ready the WRITE is issued with a correct - ReqMeta.""" - w = _StubWriterWorker.fresh() - w._logical_to_kernel_block_ids = lambda x, ratio: x - xfer_calls: list[dict[str, Any]] = [] - w._xfer_blocks_for_req = lambda **kw: xfer_calls.append(kw) - - fut: Future = Future() - w._ensure_handshake = lambda *a, **k: fut - - rd = _registration_data("req-hs", decode_engine_id="decode-engine") - _real_do_start_push_kv(w, "req-hs", ([1, 2, 3],), rd) - - # Handshake pending -> nothing issued, nothing queued, no wake. - assert xfer_calls == [] - assert w._deferred_push_inbox.qsize() == 0 - assert not w._push_writer_wake.is_set() - - # Handshake completes: request re-queued for the writer, wake set, but no - # *direct* WRITE from the callback (wrong thread for NIXL ops). - fut.set_result(({(0, 0): "agent"}, 0.0)) - assert xfer_calls == [] - assert w._push_writer_wake.is_set() - rid, blocks, reg = w._deferred_push_inbox.get_nowait() - assert (rid, blocks, reg) == ("req-hs", ([1, 2, 3],), rd) - - # Re-drive on the writer with the handshake now ready -> WRITE issued. - w._ensure_handshake = lambda *a, **k: None - _real_do_start_push_kv(w, rid, blocks, reg) - assert len(xfer_calls) == 1 - assert xfer_calls[0]["req_id"] == "req-hs" - meta = xfer_calls[0]["meta"] - assert meta.remote is not None - assert meta.remote.engine_id == "decode-engine" - # RemoteMeta.request_id is D's request id from the registration. - assert meta.remote.request_id == "req-hs" - - -def test_do_start_push_kv_drops_request_on_handshake_failure(): - """Handshake raises: the request is dropped (not re-queued, no WRITE) and - the failure is logged. Blocks are reclaimed by the lease/watchdog, matching - the old blocking behaviour.""" - w = _StubWriterWorker.fresh() - w._logical_to_kernel_block_ids = lambda x, ratio: x - xfer_calls: list[dict[str, Any]] = [] - w._xfer_blocks_for_req = lambda **kw: xfer_calls.append(kw) - failures: list[dict[str, Any]] = [] - w._log_failure = lambda **kw: failures.append(kw) - - fut: Future = Future() - w._ensure_handshake = lambda *a, **k: fut - - _real_do_start_push_kv(w, "req-fail", ([9],), _registration_data("req-fail")) - fut.set_exception(RuntimeError("handshake boom")) - - assert w._deferred_push_inbox.qsize() == 0 - assert xfer_calls == [] - assert len(failures) == 1 - assert failures[0]["failure_type"] == "push_handshake_failed" - # The handshake metric is counted once by _ensure_handshake's own - # callback (stubbed out here); the push side must not double-record it. - assert w.xfer_stats.data["num_failed_handshakes"] == [] - assert w.xfer_stats.data["num_failed_transfers"] == [] - - -def test_writer_loop_drains_deferred_push_inbox(): - """The writer loop drains ``_deferred_push_inbox`` and re-drives - ``_do_start_push_kv`` for each entry (event-driven, no polling).""" - w = _StubWriterWorker.fresh() - w.nixl_wrapper = MagicMock() - w.nixl_wrapper.get_new_notifs.return_value = {} - - processed = threading.Event() - - def _tracked(rid, blocks, rd): - w.start_push_calls.append((rid, blocks, rd)) - processed.set() - - w._do_start_push_kv = _tracked # type: ignore[method-assign] - - w._deferred_push_inbox.put( - ("req-retry", ([1, 2],), _registration_data("req-retry")) - ) - w._push_writer_wake.set() - - t = threading.Thread(target=w._push_writer_loop, daemon=True) - t.start() - try: - assert processed.wait(timeout=2.0), "writer did not drain deferred inbox" - finally: - w._push_writer_stop.set() - w._push_writer_wake.set() - t.join(timeout=2) - - assert len(w.start_push_calls) == 1 - assert w.start_push_calls[0][0] == "req-retry" - - -def _eviction_worker(engine_ttl: float) -> NixlPushConnectorWorker: - """A push worker wired to drive the base ``_ensure_handshake`` eviction - path (``_evict_stale_engines`` + ``_cleanup_remote_engine``).""" - w = _StubWriterWorker.fresh() - w._engine_ttl = engine_ttl - w._engine_last_active = {} - w._engine_clock_offset = {} - w._handshake_lock = threading.RLock() - # _cleanup_remote_engine touches these when reaping an engine. - w.nixl_wrapper = MagicMock() - w.dst_xfer_side_handles = {} - w.kv_caches_base_addr = {} - w.dst_num_blocks = {} - w.tp_mappings = {} - w.transfer_topo = None - w._logical_to_kernel_block_ids = lambda blocks, ratio: blocks - w.writes = [] - w._xfer_blocks_for_req = lambda req_id, meta: w.writes.append(req_id) - return w - - -def test_active_push_refreshes_engine_last_active(): - """The base ``_ensure_handshake`` refreshes liveness only on a new - handshake, so an active push to an already-connected engine must refresh - ``_engine_last_active`` itself or it is reaped mid-stream (S1).""" - w = _eviction_worker(engine_ttl=3600.0) - # Already-connected D -> _ensure_handshake returns None, WRITE runs inline. - w._remote_agents["decode-engine"] = {(0, 0): "agent-decode"} - stale = time.perf_counter() - 5.0 - w._engine_last_active["decode-engine"] = stale - - _real_do_start_push_kv(w, "req", ([1, 2, 3],), _registration_data("req")) - - assert w._engine_last_active["decode-engine"] > stale - assert w.writes == ["req"] - - -def test_stale_engine_evicted_on_push(): - """A push drives ``_ensure_handshake`` -> ``_evict_stale_engines``, so a D - engine silent past its TTL is reaped -- no _remote_agents / NIXL agent leak - across D scale up/down (S1).""" - w = _eviction_worker(engine_ttl=30.0) - w._remote_agents["D-old"] = {(0, 0): "agent-D-old"} - w._engine_last_active["D-old"] = time.perf_counter() - 10_000.0 - # The engine being pushed to is already connected. - w._remote_agents["decode-engine"] = {(0, 0): "agent-decode"} - - _real_do_start_push_kv(w, "req", ([1, 2, 3],), _registration_data("req")) - - assert "D-old" not in w._remote_agents - assert "D-old" not in w._engine_last_active - w.nixl_wrapper.remove_remote_agent.assert_called_once_with("agent-D-old") - - -def test_cleanup_remote_engine_pops_under_handshake_lock(): - """Regression (X1): eviction must not remove ``_remote_agents`` entries - while another thread is inside the lock's critical section (e.g. the - push writer reading the map to send a registration).""" - w = _eviction_worker(engine_ttl=30.0) - w._remote_agents["D-old"] = {(0, 0): "agent-D-old"} - - w._handshake_lock.acquire() - done = threading.Event() - - def _cleanup(): - w._cleanup_remote_engine("D-old") - done.set() - - t = threading.Thread(target=_cleanup) - t.start() - assert "D-old" in w._remote_agents - - w._handshake_lock.release() - assert done.wait(timeout=2.0) - t.join(timeout=2.0) - assert "D-old" not in w._remote_agents - w.nixl_wrapper.remove_remote_agent.assert_called_once_with("agent-D-old") - - -def _recv_metadata(remote_engine_id: str, req_id: str = "req") -> NixlConnectorMetadata: - """Build metadata with a single ``reqs_to_recv`` entry pointing at - ``remote_engine_id`` (the prefill remote D receives a push from).""" - params = { - "remote_block_ids": ([0],), - "remote_engine_id": remote_engine_id, - "remote_request_id": "p-req", - "remote_host": "localhost", - "remote_port": 1234, - "tp_size": 1, - } - meta = NixlConnectorMetadata() - meta.add_new_req_to_recv(req_id, ([0],), params) - return meta - - -def test_start_load_kv_refreshes_engine_last_active_on_d_side(): - """D receiving a push refreshes ``_engine_last_active`` for the prefill - remote, but only once that engine is connected (handshaked into - ``_remote_agents`` via heartbeats), mirroring the P-side refresh which runs - after ``_ensure_handshake``. Without it the prefill engine is reaped - mid-stream once it passes its TTL.""" - w = _StubWriterWorker.fresh() - w._send_heartbeats = lambda metadata: None - w._logical_to_kernel_block_ids = lambda x, ratio: x - # Connected prefill engine (already handshaked in). - w._remote_agents["prefill-engine"] = {(0, 0): "agent-p"} - - stale = time.perf_counter() - 5.0 - w._engine_last_active["prefill-engine"] = stale - - w.start_load_kv(_recv_metadata("prefill-engine")) - - assert w._engine_last_active["prefill-engine"] > stale - assert "req" in w._recving_metadata - - -def test_start_load_kv_skips_liveness_for_unconnected_engine(): - """If the prefill engine was never handshaked into ``_remote_agents`` (e.g. - its heartbeat handshake never succeeded), the D-side refresh must NOT record - an ``_engine_last_active`` entry -- otherwise a later eviction pass hits the - ``_remote_agents`` invariant in ``_cleanup_remote_engine`` and crashes on - the D node.""" - w = _eviction_worker(engine_ttl=30.0) - w._send_heartbeats = lambda metadata: None - - w.start_load_kv(_recv_metadata("prefill-engine")) - - # Unconnected -> no orphaned liveness entry, so eviction has nothing to trip on. - assert "prefill-engine" not in w._engine_last_active - assert "prefill-engine" not in w._remote_agents - - w._evict_stale_engines() # nothing to reap, must not raise - - -class TestPushWriterNotifs: - def test_get_new_notifs_processes_forwarded_completion_notif(self): - """Non-PUSH_REG notifs forwarded by the writer thread are drained - on the engine main thread inside ``_get_new_notifs``.""" - w = _StubWriterWorker.fresh() - # Pretend the writer thread already forwarded a completion notif - # for a request whose KV is being received. - request_id = "req-recv-1" - w._recving_metadata[request_id] = MagicMock(pp_size=1) - # Compose the standard completion notif: req_id:tp_size. - notif_msg = f"{request_id}:1".encode() - w._pending_completion_notifs.put(notif_msg) - - w._get_new_notifs() - - assert request_id in w._recving_transfers - - def test_get_transfer_results_evicts_completed_state(self): - """Transfer completion should enqueue evictions and wake the writer.""" - w = _StubWriterWorker.fresh() - - with patch.object( - NixlPushConnectorWorker.__mro__[1], - "get_transfer_results", - return_value=KVConnectorTransferResults(finished_sending={"req-done"}), - ): - results = w.get_transfer_results() - - assert results.finished_sending == {"req-done"} - assert results.finished_recving == set() - # Eviction enqueued for the writer. - evicted = [] - while True: - try: - evicted.append(w._evict_finished_inbox.get_nowait()) - except queue.Empty: - break - assert evicted == ["req-done"] - assert w._push_writer_wake.is_set() - - @staticmethod - def _pollable_worker() -> _StubWriterWorker: - """Stub worker with the extra base-worker state the real - ``get_transfer_results`` needs to poll ``_sending_transfers``.""" - w = _StubWriterWorker.fresh() - w.transfer_topo = MagicMock() - w.nixl_wrapper = MagicMock() - w.xfer_stats = MagicMock() - w._log_failure = MagicMock() # type: ignore[method-assign] - w._invalid_block_ids = queue.Queue() - w._pending_recv_notifs = {} - w._replicated_pcp_done_sending = set() - w.use_host_buffer = False - return w - - def _make_sending_req(self, w: _StubWriterWorker) -> str: - request_id = "req-send-1" - w._sending_transfers[request_id] = [101, 102] - w._reqs_to_send[request_id] = time.perf_counter() + 60 - w._reqs_to_process.add(request_id) - return request_id - - def test_failed_push_send_is_not_reported_done_sending(self): - """A failed P-side WRITE must not surface as finished sending. - - Reporting it would free P-side blocks as if the KV had been - delivered while D still waits for the data; the request must be - left to the lease / watchdog instead. - """ - w = self._pollable_worker() - request_id = self._make_sending_req(w) - w.nixl_wrapper.check_xfer_state.side_effect = ["ERR", "DONE"] - - results = w.get_transfer_results() - - assert request_id not in results.finished_sending - # Lease tracking is preserved so the watchdog can reschedule. - assert request_id in w._reqs_to_send - assert request_id in w._reqs_to_process - # Both handles were still cleaned up. - assert request_id not in w._sending_transfers - - @pytest.mark.parametrize(("pp_size", "is_hma"), [(1, False), (2, True)]) - def test_completed_push_send_releases_blocks_without_consumer_notif( - self, pp_size, is_hma - ): - """All WRITEs completing lets P reclaim its blocks without a D ACK.""" - w = self._pollable_worker() - w.pp_size = pp_size - w._is_hma_required = is_hma - request_id = self._make_sending_req(w) - w.nixl_wrapper.check_xfer_state.side_effect = ["DONE", "PROC", "DONE"] - - results = w.get_transfer_results() - - assert request_id not in results.finished_sending - assert request_id in w._reqs_to_send - assert w._sending_transfers[request_id] == [102] - assert w._evict_finished_inbox.empty() - - results = w.get_transfer_results() - assert results.finished_sending == {request_id} - assert request_id not in w._reqs_to_send - assert request_id not in w._reqs_to_process - assert w._evict_finished_inbox.get_nowait() == request_id - assert w.get_transfer_results().finished_sending == set() - assert w._evict_finished_inbox.empty() - assert w.nixl_wrapper.release_xfer_handle.call_count == 2 - w.xfer_stats.record_kv_expired_req.assert_not_called() - - @pytest.mark.parametrize("sibling_state", ["ERR", "DONE"]) - def test_push_failure_survives_until_lease_expiry(self, sibling_state): - """A later sibling completion must not turn a failed send into success.""" - w = self._pollable_worker() - request_id = self._make_sending_req(w) - w.nixl_wrapper.check_xfer_state.side_effect = ["ERR", "PROC", sibling_state] - - assert w.get_transfer_results().finished_sending == set() - assert w._sending_transfers[request_id] == [102] - assert w.get_transfer_results().finished_sending == set() - assert request_id not in w._sending_transfers - assert request_id in w._reqs_to_send - assert w._evict_finished_inbox.empty() - - w.expected_consumer_notifications_by_req = {} - w._reqs_to_send[request_id] = time.perf_counter() - 1 - assert w.get_transfer_results().finished_sending == {request_id} - assert request_id not in w._send_failures - assert w._evict_finished_inbox.get_nowait() == request_id - - @pytest.mark.parametrize("release_fails", [False, True]) - def test_push_submission_failure_is_remembered_after_sibling_success( - self, release_fails - ): - """A submission error remains a failure even if later polls succeed.""" - w = self._pollable_worker() - request_id = self._make_sending_req(w) - w.transfer_topo.block_size_ratio.return_value = 1 - w._apply_prefix_caching = MagicMock(return_value=([[1]], [[1]])) - w._compute_desc_ids = MagicMock(return_value=[0]) - w.dst_num_blocks = {w.engine_id: 8} - w.dst_region_num_blocks[w.engine_id] = [8] - w.dst_region_group_ids[w.engine_id] = [0] - w.dst_uses_region_group_mapping[w.engine_id] = False - w.nixl_wrapper.make_prepped_xfer.return_value = 101 - w.nixl_wrapper.transfer.side_effect = RuntimeError("submit") - if release_fails: - w.nixl_wrapper.release_xfer_handle.side_effect = [ - RuntimeError("release"), - None, - None, - ] - - handle = w._xfer_blocks( - read_spec=ReadSpec(0, [[1]], [[1]]), - dst_engine_id=w.engine_id, - request_id=request_id, - remote_request_id="decode-request", - local_xfer_side_handle=1, - remote_xfer_side_handle=2, - ) - assert handle == (101 if release_fails else None) - w._sending_transfers[request_id] = [102] - if handle is not None: - w._sending_transfers[request_id].append(handle) - w.nixl_wrapper.check_xfer_state.return_value = "DONE" - - assert w.get_transfer_results().finished_sending == set() - assert request_id in w._reqs_to_send - assert request_id not in w._sending_transfers - assert w._evict_finished_inbox.empty() - - @pytest.mark.parametrize("first_state", ["ERR", "DONE"]) - def test_push_completion_after_expiry_is_not_reported_twice(self, first_state): - """A WRITE finishing after lease expiry must not complete the request again.""" - w = self._pollable_worker() - request_id = self._make_sending_req(w) - w.nixl_wrapper.check_xfer_state.side_effect = [ - first_state, - "PROC", - "PROC", - "DONE", - ] - assert w.get_transfer_results().finished_sending == set() - - w.expected_consumer_notifications_by_req = {} - w._reqs_to_send[request_id] = time.perf_counter() - 1 - assert w.get_transfer_results().finished_sending == {request_id} - assert w._evict_finished_inbox.get_nowait() == request_id - - assert w.get_transfer_results().finished_sending == set() - assert request_id not in w._sending_transfers - assert request_id not in w._send_failures - assert w._evict_finished_inbox.empty() - - -# ----------------------------------------------------------------- # -# Negative / error-path tests # -# ----------------------------------------------------------------- # - - -class TestPushSchedulerNegative: - """Failure / no-op paths on the scheduler side.""" - - def test_update_state_after_alloc_no_kv_transfer_params_is_noop(self): - """Requests without kv_transfer_params must not register anything.""" - sched = make_nixl_push_scheduler() - _stub_sw_clipping(sched) - - request = MagicMock() - request.request_id = "req-no-params" - request.kv_transfer_params = None - - sched.update_state_after_alloc( - request, _BlocksMock(([1, 2, 3],)), num_external_tokens=64 - ) - - assert sched._push_pending_registrations == {} - assert sched._push_registration_deadlines == {} - assert sched._reqs_need_recv == {} - - def test_update_state_after_alloc_zero_external_tokens_does_not_register(self): - """num_external_tokens=0 should not stage a D registration.""" - sched = make_nixl_push_scheduler() - _stub_sw_clipping(sched) - - request = _make_request(request_id="req-zero-ext") - sched.update_state_after_alloc( - request, _BlocksMock(([1, 2, 3],)), num_external_tokens=0 - ) - - assert sched._push_pending_registrations == {} - assert sched._push_registration_deadlines == {} - - def test_request_finished_unfinished_status_does_not_stage(self): - """If a request is still RUNNING, request_finished must not stash - blocks for the worker (no push needed).""" - sched = make_nixl_push_scheduler() - _stub_sw_clipping(sched) - - request = _make_request( - request_id="req-running", is_d_side=False, finished=False - ) - - delay, ret = sched.request_finished(request, ([1, 2, 3],)) - - assert delay is False - assert ret is None - assert sched._finished_request_blocks == {} - assert sched._newly_finished_push_blocks == {} - - def test_request_finished_empty_blocks_does_not_arm_lease(self): - """Empty block-id groups should still complete cleanly without - arming the lease/finished maps.""" - sched = make_nixl_push_scheduler() - _stub_sw_clipping(sched) - - request = _make_request(request_id="req-empty", is_d_side=False) - delay, ret = sched.request_finished(request, ((),)) - - assert delay is False - assert ret is not None - assert "req-empty" not in sched._finished_request_blocks - assert "req-empty" not in sched._newly_finished_push_blocks - assert "req-empty" not in sched._reqs_need_send - - def test_update_connector_output_unknown_request_is_noop(self): - """Idempotent cleanup: clearing a request that was never staged - must not raise or mutate other state.""" - sched = make_nixl_push_scheduler() - _stub_sw_clipping(sched) - - # Stage one real request to ensure it's NOT touched. - live = _make_request(request_id="req-live", is_d_side=False) - sched.request_finished(live, ([1],)) - - sched.update_connector_output( - KVConnectorOutput( - finished_sending={"unknown-1"}, - finished_recving={"unknown-2"}, - invalid_block_ids=set(), - ) - ) - - # Live entry untouched. - assert "req-live" in sched._finished_request_blocks - - -class TestPushWriterNegative: - """Failure / drop / idempotence paths in the writer thread.""" - - def test_pop_matching_registration_returns_none_when_empty(self): - w = _StubWriterWorker.fresh() - assert w._pop_matching_registration("nope") is None - - def test_pop_matching_finished_blocks_returns_none_when_empty(self): - w = _StubWriterWorker.fresh() - assert w._pop_matching_finished_blocks("nope") is None - - def test_pop_matching_registration_no_match_when_base_ids_differ(self): - """A registration whose base id (after stripping the random suffix) - does NOT match the lookup request_id must not be popped.""" - w = _StubWriterWorker.fresh() - # Two unrelated requests: different base UUIDs, so stripping the - # trailing ``-<8 hex>`` suffix still yields different base ids. - unrelated_d = "cmpl-aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa-0-11111111" - lookup = "cmpl-bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb-0-22222222" - assert get_base_request_id(unrelated_d) != get_base_request_id(lookup) - - w._pending_d_registrations[unrelated_d] = _registration_data(unrelated_d) - result = w._pop_matching_registration(lookup) - assert result is None - # Original entry untouched. - assert unrelated_d in w._pending_d_registrations - - def test_handle_push_reg_with_non_dict_payload_is_dropped(self, caplog): - """msgpack-encoded non-dict payload (e.g. a list) should be - dropped without raising.""" - caplog.set_level( - logging.CRITICAL, - logger=("vllm.distributed.kv_transfer.kv_connector.v1.nixl.push_worker"), - ) - w = _StubWriterWorker.fresh() - bad = PUSH_REG_NOTIF_PREFIX + msgspec.msgpack.encode([1, 2, 3]) - w._handle_push_reg_notif(bad) - assert w._pending_d_registrations == {} - assert w.start_push_calls == [] - - def test_handle_push_reg_with_non_string_request_id_is_dropped(self, caplog): - """request_id must be a str; integers, None, etc. must drop.""" - caplog.set_level( - logging.CRITICAL, - logger=("vllm.distributed.kv_transfer.kv_connector.v1.nixl.push_worker"), - ) - w = _StubWriterWorker.fresh() - for bogus_rid in (123, None, 4.5, b"bytes-not-str"): - payload = _registration_data("placeholder") - payload["request_id"] = bogus_rid - notif = PUSH_REG_NOTIF_PREFIX + msgspec.msgpack.encode(payload) - w._handle_push_reg_notif(notif) - assert w._pending_d_registrations == {} - assert w.start_push_calls == [] - - def test_handle_push_reg_idempotent_for_same_request_id(self): - """Receiving the same PUSH_REG twice (e.g. P retries after a - flake) keeps the entry staged exactly once and never fires.""" - w = _StubWriterWorker.fresh() - notif = PUSH_REG_NOTIF_PREFIX + msgspec.msgpack.encode( - _registration_data("req-dup") - ) - w._handle_push_reg_notif(notif) - w._handle_push_reg_notif(notif) - assert "req-dup" in w._pending_d_registrations - assert len(w._pending_d_registrations) == 1 - assert w.start_push_calls == [] - - def test_get_transfer_results_enqueues_each_completed_request(self): - """``get_transfer_results`` must enqueue every completed request - in ``done_sending`` so the writer can drop stale matching state. - Unlike the happy-path test, this verifies the *cardinality*: N - completed requests -> N evictions, in order.""" - w = _StubWriterWorker.fresh() - with patch.object( - NixlPushConnectorWorker.__mro__[1], - "get_transfer_results", - return_value=KVConnectorTransferResults( - finished_sending={"req-1", "req-2", "req-3"} - ), - ): - results = w.get_transfer_results() - assert results.finished_sending == {"req-1", "req-2", "req-3"} - - evicted: list[str] = [] - while True: - try: - evicted.append(w._evict_finished_inbox.get_nowait()) - except queue.Empty: - break - assert sorted(evicted) == ["req-1", "req-2", "req-3"] - - def test_empty_transfer_results_do_not_enqueue_eviction(self): - """If there's nothing newly done, no eviction should be enqueued. - The wake event IS still set because ``get_finished`` always wakes - the writer to drain notifs.""" - w = _StubWriterWorker.fresh() - with patch.object( - NixlPushConnectorWorker.__mro__[1], - "get_transfer_results", - return_value=KVConnectorTransferResults(), - ): - results = w.get_transfer_results() - assert results.finished_sending == set() - assert results.finished_recving == set() - assert w._evict_finished_inbox.qsize() == 0 - # Wake set so the writer drains NIXL notifs even when idle. - assert w._push_writer_wake.is_set() - - def test_failed_push_transfer_does_not_evict_writer_state(self): - w = _StubWriterWorker.fresh() - w.nixl_wrapper = MagicMock() - w.nixl_wrapper.check_xfer_state.return_value = "ERR" - w.xfer_stats = MagicMock() - w._log_failure = MagicMock() - w._sending_transfers["req-failed"] = [17] - w._reqs_to_send["req-failed"] = time.perf_counter() + 30 - w._reqs_to_process.add("req-failed") - - with patch.object( - NixlPushConnectorWorker.__mro__[1], - "get_transfer_results", - return_value=KVConnectorTransferResults(), - ): - results = w.get_transfer_results() - - assert results.finished_sending == set() - assert results.finished_recving == set() - assert "req-failed" not in w._sending_transfers - assert "req-failed" in w._reqs_to_send - assert "req-failed" in w._reqs_to_process - assert w._evict_finished_inbox.empty() - w.nixl_wrapper.release_xfer_handle.assert_called_once_with(17) - - def test_get_new_notifs_unknown_request_is_logged_and_skipped(self, caplog): - """A completion notif for a request the worker doesn't know - about should be logged but not crash.""" - caplog.set_level( - logging.CRITICAL, - logger=("vllm.distributed.kv_transfer.kv_connector.v1.nixl.push_worker"), - ) - w = _StubWriterWorker.fresh() - # Forward a completion notif for an unknown request_id. - w._pending_completion_notifs.put(b"never-heard-of-you:1") - - w._get_new_notifs() - # Did not register anywhere. - assert "never-heard-of-you" not in w._recving_transfers - assert w.xfer_stats.data["num_notifications_after_expiry"] == [1] - - def test_start_load_kv_with_empty_metadata_is_noop(self): - """Empty metadata must not wake the writer or enqueue anything.""" - w = _StubWriterWorker.fresh() - w._send_heartbeats = lambda metadata: None - w._logical_to_kernel_block_ids = lambda x, ratio: x - - meta = NixlConnectorMetadata() - w.start_load_kv(meta) - - assert w._reg_send_inbox.qsize() == 0 - assert w._finished_blocks_inbox.qsize() == 0 - # Wake should NOT be set if there was nothing to push. - assert not w._push_writer_wake.is_set() - - def test_get_new_notifs_extends_lease_on_heartbeat(self): - """``HB:`` notifs forwarded by the writer thread must extend the - leases of tracked P-side requests on the engine main thread, and - ignore request IDs that aren't being tracked.""" - w = _StubWriterWorker.fresh() - # _handle_heartbeat reads ``self._lease_extension`` (set in the - # real ``__init__``). - w._lease_extension = 10 - - # Tracked P-side requests with a lease about to expire. - old_expiry = time.perf_counter() - 5.0 - w._reqs_to_send["req-a"] = old_expiry - w._reqs_to_send["req-b"] = old_expiry - - # Forwarded heartbeat covers a tracked request, an unknown one, - # and another tracked one. - w._pending_completion_notifs.put(b"HB:req-a,req-unknown,req-b") - - w._get_new_notifs() - - # Tracked leases were renewed strictly forward in time. - now = time.perf_counter() - for rid in ("req-a", "req-b"): - assert w._reqs_to_send[rid] > old_expiry - # New expiry must be roughly now + _lease_extension. - assert w._reqs_to_send[rid] >= now - # Unknown request must not be inserted by the heartbeat path. - assert "req-unknown" not in w._reqs_to_send - - -# ----------------------------------------------------------------- # -# Pipeline-parallel producer (push-mode PP-disagg) # -# ----------------------------------------------------------------- # - - -class TestPushPipelineParallel: - """PP-sharded producer: per-stage completion counting, pp_size plumbing, - and remote-region slicing.""" - - def test_completion_waits_for_one_notif_per_pp_stage(self): - """Each PP stage WRITEs its own layers and sends one notif; D must - collect pp_size notifs before reporting the recv done.""" - w = _StubWriterWorker.fresh() - request_id = "req-pp-2" - w._recving_metadata[request_id] = MagicMock(pp_size=2) - notif = f"{request_id}:1".encode() - - # First stage: counted, not yet done. - w._pending_completion_notifs.put(notif) - w._get_new_notifs() - assert request_id not in w._recving_transfers - assert w.consumer_notification_counts_by_req[request_id] == 1 - - # Second (final) stage: now reported done. - w._pending_completion_notifs.put(notif) - w._get_new_notifs() - assert request_id in w._recving_transfers - assert request_id not in w.consumer_notification_counts_by_req - - def test_req_meta_reads_pp_size_from_kv_transfer_params(self): - """D learns the producer's pp_size from kv_transfer_params (forwarded - by the proxy) and defaults to 1 when absent.""" - metadata = NixlConnectorMetadata() - params = { - "remote_block_ids": ([0],), - "remote_engine_id": "p-engine", - "remote_request_id": "p-req", - "remote_host": "localhost", - "remote_port": 1234, - "tp_size": 1, - "pp_size": 2, - } - metadata.add_new_req_to_recv("req", ([0],), params) - assert metadata.reqs_to_recv["req"].pp_size == 2 - - params.pop("pp_size") - metadata.add_new_req_to_recv("req-default", ([0],), params) - assert metadata.reqs_to_recv["req-default"].pp_size == 1 - - def test_add_remote_agent_slices_remote_regions_to_local_pp_window(self): - """With PP>1 the producer registered regions for the full model, but - this worker holds only a contiguous layer slice; add_remote_agent - trims the remote region list to [offset : offset + num_local_regions] - before building descriptors. We stop right after the slice via a - sentinel on the next collaborator call.""" - block_len = 4096 * 16 - w = _StubWriterWorker.fresh() # seeds writer-thread state for teardown - w.pp_size = 2 - w._remote_region_offset = 2 # this worker owns layers [2, 4) - w.block_len_per_layer = [block_len, block_len] # 2 local layers - w.nixl_wrapper = MagicMock() - - class _StopAfterSlice(RuntimeError): - pass - - w.transfer_topo = MagicMock() - w.transfer_topo.register_remote_engine.side_effect = _StopAfterSlice() - - meta = NixlAgentMetadata( - engine_id="p-engine", - agent_metadata=b"agent", - kv_caches_base_addr=[10, 11, 12, 13], # full 4-layer model - device_id=0, - num_blocks=4, - block_lens=[block_len] * 4, - # Non-interleaved: consecutive blocks abut, so stride == block length. - block_strides=[block_len] * 4, - kv_cache_layout="HND", - block_size=16, - ssm_sizes=(0, 0), - attn_backend_name="FLASH_ATTN", - physical_blocks_per_logical_kv_block=1, - ) - - with pytest.raises(_StopAfterSlice): - w.add_remote_agent(meta, remote_tp_rank=0, remote_tp_size=1) - - # Sliced down to this worker's layer window (regions [2:4]). - assert meta.kv_caches_base_addr == [12, 13] - assert meta.block_lens == [block_len, block_len] - - -class TestPushWriterMlaReplication: - """MLA latent KV is replicated across D's TP ranks, so when D_TP > P_TP - (``tp_ratio < 0``) the producer must *WRITE* the latent into every D rank - it handshook -- not write one and merely notify the rest, which would - leave the un-written ranks decoding against stale KV.""" - - @staticmethod - def _mla_worker_writing_to(d_ranks): - from types import SimpleNamespace - - from vllm.distributed.kv_transfer.kv_connector.v1.nixl.tp_mapping import ( - TPMapping, - ) - - engine_id = "decode-engine" - w = _StubWriterWorker.fresh() - w.use_mla = True - w.nixl_wrapper = MagicMock() - w.transfer_topo = MagicMock() - w.transfer_topo.get_engine_info.return_value = SimpleNamespace( - remote_tp_size=len(d_ranks), - remote_block_size=16, - remote_physical_blocks_per_logical=1, - ) - # D_TP > P_TP => negative ratio (one P rank feeds |ratio| D ranks). - w.transfer_topo.tp_ratio.return_value = -len(d_ranks) - # The tp-mapping collapses MLA to a single source rank (correct for - # the pull/read direction); the push path must fan it back out. - w.tp_mappings = { - engine_id: TPMapping( - source_ranks_per_group=((0,),), - all_source_ranks=(0,), - rank_to_attention_slot={0: 0}, - rank_offset_factor=0, - ) - } - w._logical_to_kernel_block_ids = lambda block_ids, ratio: block_ids - w.dst_region_group_ids[engine_id] = [0] - w.dst_xfer_side_handles = {engine_id: {r: 1000 + r for r in d_ranks}} - w.src_xfer_handles_by_block_size = {16: 2000} - w._remote_agents = {engine_id: {(0, r): f"agent-{r}" for r in d_ranks}} - return w, engine_id - - def test_mla_hetero_tp_writes_every_d_rank(self): - from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import ( - RemoteMeta, - ReqMeta, - ) - - w, engine_id = self._mla_worker_writing_to(d_ranks=(0, 1)) - - written: list[int] = [] - - def _fake_xfer(**kw): - rank = kw["read_spec"].remote_rank - written.append(rank) - return 1000 + rank # in-flight handle, as the real method returns - - w._xfer_blocks = _fake_xfer - - meta = ReqMeta( - local_block_ids=([100, 101],), - local_physical_block_ids=([100, 101],), - tp_size=1, - remote=RemoteMeta( - block_ids=([200, 201],), - host="", - port=0, - engine_id=engine_id, - request_id="d-req", - ), - ) - - w._xfer_blocks_for_req(req_id="p-req", meta=meta) - - # Every handshook D rank must receive a real WRITE of the latent... - assert sorted(written) == [0, 1] - # ...and no rank may be fobbed off with a bare completion notif. - assert w.nixl_wrapper.send_notif.call_count == 0 - # All of the request's WRITE handles must be tracked together, so the - # engine thread never sees a partial set and double-frees the request. - assert sorted(w._sending_transfers["p-req"]) == [1000, 1001] - - def test_csa_linear_writes_every_group_to_each_target(self): - from types import SimpleNamespace - - from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import ( - RemoteMeta, - ReqMeta, - ) - from vllm.distributed.kv_transfer.kv_connector.v1.nixl.tp_mapping import ( - TPMapping, - ) - - engine_id = "decode-engine" - w = _StubWriterWorker.fresh() - w.world_size = 2 - w.use_mla = False - w._has_mamba = True - w._is_csa_linear = True - w.nixl_wrapper = MagicMock() - w.transfer_topo = MagicMock() - w.transfer_topo.get_engine_info.return_value = SimpleNamespace( - remote_tp_size=4, - remote_block_size=16, - remote_physical_blocks_per_logical=1, - ) - w.transfer_topo.tp_ratio.return_value = -2 - w.transfer_topo.handshake_target_ranks.return_value = (0, 1) - w.tp_mappings = { - engine_id: TPMapping( - source_ranks_per_group=((0,), (0,), (0, 1), (0, 1)), - all_source_ranks=(0, 1), - rank_to_attention_slot={0: 0, 1: 0}, - rank_offset_factor=0, - ) - } - w._logical_to_kernel_block_ids = lambda block_ids, ratio: block_ids - w.region_group_ids = [0, 1, 2, 3] - w.dst_region_group_ids[engine_id] = [0, 1, 2, 3] - w.dst_xfer_side_handles = {engine_id: {0: 1000, 1: 1001}} - w.src_xfer_handles_by_tp_ratio = {(-2, 16): [2000, 2001]} - - writes = [] - - def _fake_xfer(**kwargs): - writes.append(kwargs) - return 3000 + kwargs["read_spec"].remote_rank - - w._xfer_blocks = _fake_xfer - block_ids = ([10], [20], [30], [40]) - meta = ReqMeta( - local_block_ids=block_ids, - local_physical_block_ids=block_ids, - tp_size=2, - remote=RemoteMeta( - block_ids=([50], [60], [70], [80]), - host="", - port=0, - engine_id=engine_id, - request_id="d-req", - ), - ) - - w._xfer_blocks_for_req(req_id="p-req", meta=meta) - - assert [write["read_spec"].remote_rank for write in writes] == [0, 1] - assert [write["read_spec"].local_block_ids for write in writes] == [ - list(block_ids), - list(block_ids), - ] - assert [write["local_xfer_side_handle"] for write in writes] == [2000, 2001] - assert sorted(w._sending_transfers["p-req"]) == [3000, 3001] - - -class TestPushPrefixCaching: - """Partial prefix-cache hit on D: D preallocates only its *uncomputed* - blocks, so on a partial hit it registers fewer blocks than P's full - sequence. P must WRITE only the *tail* of its sequence into D's slots -- - mirroring pull-mode ``_apply_prefix_caching`` (end-trim), never a front-trim - which would write P's cached prefix into D's uncomputed suffix slots. - - The trim runs inside ``_xfer_blocks``, so these tests drive the real - ``_xfer_blocks_for_req`` path and stub only the NIXL WRITE. With one region - and 1:1 ratios ``_compute_desc_ids`` is the identity map, so the descs - captured from ``make_prepped_xfer`` are exactly the (trimmed) block IDs. - """ - - @staticmethod - def _worker_driving_xfer(engine_id: str = "decode-engine"): - from types import SimpleNamespace - - from vllm.distributed.kv_transfer.kv_connector.v1.nixl.tp_mapping import ( - TPMapping, - ) - - w = _StubWriterWorker.fresh() - w._has_mamba = False - w.use_mla = False - w.block_size = 16 - w.num_regions = 1 - w._physical_blocks_per_logical_kv_block = 1 - - w.transfer_topo = MagicMock() - w.transfer_topo.get_engine_info.return_value = SimpleNamespace( - remote_physical_blocks_per_logical=1, - remote_block_size=16, - remote_tp_size=1, - ) - w.transfer_topo.tp_ratio.return_value = 1 - w.transfer_topo.block_size_ratio.return_value = 1 - w.tp_mappings = { - engine_id: TPMapping( - source_ranks_per_group=((0,),), - all_source_ranks=(0,), - rank_to_attention_slot={0: 0}, - rank_offset_factor=0, - ) - } - w.dst_num_blocks = {engine_id: 10_000, w.engine_id: 10_000} - w.dst_region_num_blocks = { - engine_id: [10_000], - w.engine_id: [10_000], - } - w.dst_region_group_ids = {engine_id: [0], w.engine_id: [0]} - w.dst_uses_region_group_mapping = { - engine_id: False, - w.engine_id: False, - } - w.dst_xfer_side_handles = {engine_id: {0: 5000}} - w.src_xfer_handles_by_block_size = {16: 2000} - - # Stub only the NIXL WRITE; kernel expansion, prefix-cache trim, desc - # computation and count assertions all run for real. - w.nixl_wrapper = MagicMock() - w.nixl_wrapper.make_prepped_xfer.return_value = 7 - w._ensure_handshake = lambda *a, **k: None - w._logical_to_kernel_block_ids = lambda x, ratio: x - return w, engine_id - - @staticmethod - def _written_block_ids(w) -> tuple[list[int], list[int]]: - """Return the (local, remote) block IDs handed to the NIXL WRITE.""" - args, _ = w.nixl_wrapper.make_prepped_xfer.call_args - # ("WRITE", local_handle, local_descs, remote_handle, remote_descs) - return list(args[2]), list(args[4]) - - def test_partial_prefix_hit_end_trims_producer_blocks(self): - """D registered only its 2 uncomputed suffix blocks; P finished the - full 5-block sequence. P must WRITE its LAST 2 blocks into D's slots.""" - w, _ = self._worker_driving_xfer() - reg = _registration_data("req-pc", local_block_ids=([500, 501],)) - - NixlPushConnectorWorker._do_start_push_kv( - w, "req-pc", ([10, 11, 12, 13, 14],), reg - ) - - local, remote = self._written_block_ids(w) - # End-trim (suffix), NOT front-trim: [13, 14], not [10, 11]. - assert local == [13, 14] - assert remote == [500, 501] - - def test_no_prefix_hit_leaves_blocks_untrimmed(self): - """Equal counts (no prefix cache hit on D): nothing is trimmed.""" - w, _ = self._worker_driving_xfer() - reg = _registration_data("req-full", local_block_ids=([500, 501, 502],)) - - NixlPushConnectorWorker._do_start_push_kv(w, "req-full", ([10, 11, 12],), reg) - - local, remote = self._written_block_ids(w) - assert local == [10, 11, 12] - assert remote == [500, 501, 502] - - @pytest.mark.parametrize("layer_name_routing", [False, True]) - def test_different_region_groups_require_layer_routing(self, layer_name_routing): - """A layer mapped to a pooled remote region can still receive its blocks.""" - w, engine_id = self._worker_driving_xfer() - w.dst_region_group_ids[engine_id] = [-1] - if layer_name_routing: - w._transfer_layer_group_ids = (0,) - reg = _registration_data("req-groups", local_block_ids=([500, 501],)) - - if not layer_name_routing: - with pytest.raises(NotImplementedError, match="cache-group layouts"): - NixlPushConnectorWorker._do_start_push_kv( - w, "req-groups", ([10, 11],), reg - ) - w.nixl_wrapper.make_prepped_xfer.assert_not_called() - return - - NixlPushConnectorWorker._do_start_push_kv(w, "req-groups", ([10, 11],), reg) - - local, remote = self._written_block_ids(w) - assert local == [10, 11] - assert remote == [500, 501] - - -def _agent_metadata( - region_layers: list[list[str]], - base_addresses: list[int], - block_lens: list[int], - block_strides: list[int] | None = None, -) -> NixlAgentMetadata: - return NixlAgentMetadata( - engine_id="remote-engine", - agent_metadata=b"agent", - kv_caches_base_addr=base_addresses, - device_id=7, - num_blocks=2, - block_lens=block_lens, - block_strides=list(block_lens if block_strides is None else block_strides), - kv_cache_layout="LBHNC", - block_size=16, - ssm_sizes=(0, 0), - attn_backend_name="FLASH_ATTN", - physical_blocks_per_logical_kv_block=1, - region_members=region_layers, - ) - - -def _layer_routing_worker( - region_layers: list[list[str]], - group_by_layer: dict[str, int], - pp_size: int = 2, - is_hma: bool = True, -) -> _StubWriterWorker: - worker = _StubWriterWorker.fresh() - worker.pp_size = pp_size - worker.dcp_size = 1 - worker.block_size = 16 - worker._has_mamba = False - worker._is_hma_required = is_hma - worker.kv_cache_config = MagicMock(transfer_group_index_by_layer=group_by_layer) - worker.transfer_topo = MagicMock() - worker._set_region_layers(region_layers) - return worker - - -@pytest.mark.parametrize( - ("region_num_blocks", "expected"), - [(None, [1, 2, 15, 21, 22]), ([4, 7, 5], [1, 2, 9, 12, 13])], -) -def test_layer_group_ids_route_descriptor_blocks(region_num_blocks, expected): - worker = _layer_routing_worker( - [["a", "a.swa"], ["b"]], {"a": 0, "a.swa": 1, "b": 0} - ) - desc_ids = worker._compute_desc_ids( - block_ids=[[1, 2], [5]], - dst_num_blocks=10, - block_size_ratio=None, - physical_blocks_per_logical=1, - region_num_blocks=region_num_blocks, - ) - assert desc_ids.tolist() == expected - - -def test_layer_metadata_round_trip(): - metadata = _agent_metadata([["L0", "L1"]], [0x10000], [256]) - metadata.region_num_blocks = [4] - metadata.region_group_ids = [-1] - metadata.region_names = ["L0"] - metadata.region_mem_types = ["VRAM"] - - encoded = msgspec.msgpack.encode(metadata) - assert msgspec.msgpack.Decoder(NixlAgentMetadata).decode(encoded) == metadata - - -@pytest.mark.parametrize( - "layouts, error", - [ - ({"L0": (-1, 64)}, "escapes its block"), - ({"L0": (0, 0)}, "escapes its block"), - ({"L0": (200, 64)}, "escapes its block"), - ({"L1": (0, 64)}, "has no layout"), - ({"L0": (0, 32)}, "page sizes must match"), - ], -) -def test_packed_layer_alignment_rejects_invalid_page_layout(layouts, error): - worker = _layer_routing_worker([["L0"]], {"L0": 0}) - worker.block_len_per_layer = [64] - metadata = _agent_metadata([["L0", "L1"]], [0x10000], [256]) - metadata.packed_member_layouts = layouts - with pytest.raises(AssertionError, match=error): - worker._align_remote_regions_by_layer(metadata) - - -def test_layer_alignment_preserves_nonpacked_regions_alongside_packed_layers(): - worker = _layer_routing_worker([["a"], ["b"]], {"a": 0, "b": 1}) - worker.block_len_per_layer = [64, 128] - metadata = _agent_metadata( - [["a", "other"], ["b"]], [0x1000, 0x2000], [192, 128], [256, 128] - ) - metadata.packed_member_layouts = {"a": (64, 64), "other": (128, 64)} - - worker._align_remote_regions_by_layer(metadata) - - assert metadata.kv_caches_base_addr == [0x1040, 0x2000] - assert metadata.block_lens == [64, 128] - assert metadata.block_strides == [256, 128] - - -def test_layer_identity_gate_preserves_the_non_hma_path(): - assert _layer_routing_worker([["a"]], {"a": 0})._transfer_layer_region_indices == ( - 0, - ) - - # A base worker (pull) never routes by layer name. - pull = object.__new__(NixlBaseConnectorWorker) - pull.pp_size = 2 - pull._transfer_layer_names = () - pull._transfer_layer_region_indices = () - pull._transfer_layer_group_ids = () - pull._set_region_layers([["a"]]) - assert pull._transfer_layer_region_indices == () - - # A non-HMA local layout does not require layer-name routing. - assert ( - _layer_routing_worker( - [["a"]], {"a": 0}, is_hma=False - )._transfer_layer_region_indices - == () - ) - - # PP=1 has congruent local/remote regions and keeps the region route even - # when its allocator is hybrid. - assert ( - _layer_routing_worker( - [["a"]], {"a": 0}, pp_size=1 - )._transfer_layer_region_indices - == () - ) - - -def test_layer_alignment_fails_loud_when_remote_omits_layers(): - # Falling back to region-index routing here would silently transfer stale - # KV, so an unannounced peer must fail the handshake instead. - worker = _layer_routing_worker([["a"]], {"a": 0}) - with pytest.raises(AssertionError, match="no region_members"): - worker._align_remote_regions_by_layer(_agent_metadata([], [0xA000], [128])) - - -def test_layer_alignment_expands_pooled_regions(): - worker = _layer_routing_worker( - [["a", "a.swa"], ["b"]], {"a": 0, "a.swa": 1, "b": 0} - ) - assert worker._transfer_layer_names == ("a", "a.swa", "b") - assert worker._transfer_layer_region_indices == (0, 0, 1) - assert worker._transfer_layer_group_ids == (0, 1, 0) - - metadata = _agent_metadata([["a", "a.swa"], ["b"]], [0xA000, 0xB000], [128, 128]) - metadata.region_num_blocks = [4, 6] - metadata.region_group_ids = [-1, 0] - metadata.region_names = ["a", "b"] - metadata.region_mem_types = ["VRAM", "VRAM"] - worker._align_remote_regions_by_layer(metadata) - - assert metadata.kv_caches_base_addr == [0xA000, 0xA000, 0xB000] - assert metadata.block_lens == [128, 128, 128] - assert metadata.block_strides == [128, 128, 128] - assert metadata.region_members == [["a"], ["a.swa"], ["b"]] - assert metadata.region_num_blocks == [4, 4, 6] - assert metadata.region_group_ids == [-1, -1, 0] - assert metadata.region_names == ["a", "a.swa", "b"] - assert metadata.region_mem_types == ["VRAM", "VRAM", "VRAM"] - - -def test_layer_alignment_filters_and_reorders_a_pp_stage(): - worker = _layer_routing_worker([["l2"], ["l3"]], {"l2": 0, "l3": 1}) - metadata = _agent_metadata( - [["l2"], ["l0"], ["l3"], ["l1"]], - [0xC000, 0xA000, 0xD000, 0xB000], - [65536, 65536, 32768, 32768], - [131072, 131072, 65536, 65536], - ) - metadata.region_num_blocks = [4, 6, 8, 10] - metadata.region_group_ids = [0, 0, 1, 1] - metadata.region_names = ["l2", "l0", "l3", "l1"] - metadata.region_mem_types = ["VRAM"] * 4 - - worker._align_remote_regions_by_layer(metadata) - - assert worker._transfer_layer_region_indices == (0, 1) - assert worker._transfer_layer_group_ids == (0, 1) - assert metadata.kv_caches_base_addr == [0xC000, 0xD000] - assert metadata.block_lens == [65536, 32768] - assert metadata.block_strides == [131072, 65536] - assert metadata.region_num_blocks == [4, 8] - assert metadata.region_group_ids == [0, 1] - assert metadata.region_names == ["l2", "l3"] - assert metadata.region_mem_types == ["VRAM", "VRAM"] - - -@pytest.mark.parametrize( - ("local_num_blocks", "remote_num_blocks"), - [([4, 4], [4, 4, 4]), ([4, 6], [3, 5, 7])], -) -def test_layer_descriptors_pair_layers_across_asymmetric_pp_split( - local_num_blocks, remote_num_blocks -): - """A PP split can leave each stage a different mix of attention types. - - Stage 1 owns L3 (full) pooled with L4 (sliding) in region 0, and L5 (full) - alone in region 1. The consumer holds every layer and pools them - differently, so region indices cannot be paired positionally. - """ - worker = _layer_routing_worker([["L3", "L4"], ["L5"]], {"L3": 0, "L4": 1, "L5": 0}) - worker.block_len_per_layer = [128, 128] - worker.block_stride_per_layer = [256, 512] - worker._region_is_mla = [False, False] - worker.num_blocks = 4 - worker.region_num_blocks = local_num_blocks - worker.device_id = 0 - - consumer = _agent_metadata( - [["L0", "L1"], ["L3", "L2"], ["L5", "L4"]], - [0xA000, 0xB000, 0xC000], - [128, 128, 128], - [256, 512, 1024], - ) - consumer.num_blocks = 4 - consumer.region_num_blocks = remote_num_blocks - worker._align_remote_regions_by_layer(consumer) - - # L3 -> remote region 1, L4 -> 2, L5 -> 2. Pairing by index would have sent - # L5 to region 1 and L4 to region 2's sibling. - assert consumer.kv_caches_base_addr == [0xB000, 0xC000, 0xC000] - - plan = TPMapping(((0,), (0,)), (0,), {0: 0}, 0) - # Builders emit one strided run per region; expand to per-block descriptors. - local_descs = expand_strided_descs( - worker._build_fa_local([0x1000, 0x2000], block_size_ratio=1) - ) - remote_descs = expand_strided_descs( - worker._build_fa_remote(plan, consumer, block_size_ratio=1) - ) - local_counts = [local_num_blocks[i] for i in worker._transfer_layer_region_indices] - local_ids = worker._compute_desc_ids( - [[1, 2], [3]], 4, None, 1, region_num_blocks=local_counts - ) - remote_ids = worker._compute_desc_ids( - [[0, 1], [2]], 4, None, 1, region_num_blocks=consumer.region_num_blocks - ) - assert len(local_descs) == sum(local_counts) - assert len(remote_descs) == sum(consumer.region_num_blocks) - - # Each pair addresses the same layer's blocks despite different pooling - # and strides. L3 and L5 use group 0; L4 uses group 1. - assert local_descs[local_ids].tolist() == [ - [0x1100, 128, 0], - [0x1200, 128, 0], - [0x1300, 128, 0], - [0x2200, 128, 0], - [0x2400, 128, 0], - ] - assert remote_descs[remote_ids].tolist() == [ - [0xB000, 128, 7], - [0xB200, 128, 7], - [0xC800, 128, 7], - [0xC000, 128, 7], - [0xC400, 128, 7], - ] - - -def test_layer_alignment_is_canonical_across_remote_orderings(): - local, groups = [["x"], ["y"]], {"x": 0, "y": 1} - rank0 = _agent_metadata([["x"], ["y"]], [0x1000, 0x2000], [64, 128], [256, 512]) - rank1 = _agent_metadata([["y"], ["x"]], [0x2000, 0x1000], [128, 64], [512, 256]) - - _layer_routing_worker(local, groups)._align_remote_regions_by_layer(rank0) - _layer_routing_worker(local, groups)._align_remote_regions_by_layer(rank1) - - assert rank0.kv_caches_base_addr == rank1.kv_caches_base_addr == [0x1000, 0x2000] - assert rank0.block_lens == rank1.block_lens == [64, 128] - assert rank0.block_strides == rank1.block_strides == [256, 512] - - -def test_layer_alignment_is_idempotent(): - worker = _layer_routing_worker( - [["a", "a.swa"], ["b"]], {"a": 0, "a.swa": 1, "b": 0} - ) - metadata = _agent_metadata([["a", "a.swa"], ["b"]], [0xA000, 0xB000], [128, 128]) - metadata.region_num_blocks = [4, 6] - metadata.region_group_ids = [-1, 0] - metadata.region_names = ["a", "b"] - metadata.region_mem_types = ["VRAM", "VRAM"] - - worker._align_remote_regions_by_layer(metadata) - aligned = msgspec.msgpack.encode(metadata) - worker._align_remote_regions_by_layer(metadata) - - assert msgspec.msgpack.encode(metadata) == aligned - - -def test_layer_alignment_rejects_missing_local_layer(): - worker = _layer_routing_worker([["l0"], ["l1"]], {"l0": 0, "l1": 1}) - metadata = _agent_metadata([["l0"]], [0xA000], [128]) - - with pytest.raises(AssertionError, match="missing locally owned layers"): - worker._align_remote_regions_by_layer(metadata) - - -def test_layer_alignment_rejects_duplicate_remote_layer(): - worker = _layer_routing_worker([["a"]], {"a": 0}) - metadata = _agent_metadata([["a"], ["a"]], [0xA000, 0xB000], [128, 128]) - - with pytest.raises(AssertionError, match="in multiple regions"): - worker._align_remote_regions_by_layer(metadata) - - -def test_layer_alignment_rejects_inconsistent_remote_metadata(): - worker = _layer_routing_worker([["a"]], {"a": 0}) - metadata = _agent_metadata([["a"], ["b"]], [0xA000], [128]) - - with pytest.raises(AssertionError, match="lengths disagree"): - worker._align_remote_regions_by_layer(metadata) - - -@pytest.mark.parametrize( - "field", - ["region_num_blocks", "region_group_ids", "region_names", "region_mem_types"], -) -def test_layer_alignment_rejects_inconsistent_region_geometry(field): - worker = _layer_routing_worker([["a"]], {"a": 0}) - metadata = _agent_metadata([["a"], ["b"]], [0xA000, 0xB000], [128, 128]) - setattr(metadata, field, []) - original = msgspec.msgpack.encode(metadata) - - with pytest.raises(AssertionError, match="lengths disagree"): - worker._align_remote_regions_by_layer(metadata) - assert msgspec.msgpack.encode(metadata) == original - - -def test_set_region_layers_rejects_duplicate_local_layer(): - with pytest.raises(AssertionError, match="spans multiple NIXL regions"): - _layer_routing_worker([["a"], ["a"]], {"a": 0}) - - -def test_set_region_layers_rejects_layer_outside_any_kv_group(): - with pytest.raises(AssertionError, match="outside any local group"): - _layer_routing_worker([["a"], ["b"]], {"a": 0}) - - -@pytest.mark.parametrize( - ("local_block_size", "remote_block_size", "remote_tp_size", "error"), - [ - (32, 16, 1, "identical P/D block sizes"), - (16, 32, 1, "identical P/D block sizes"), - (16, 16, 2, "decode TP greater"), - ], -) -def test_layer_handshake_rejects_unsupported_geometry( - local_block_size: int, remote_block_size: int, remote_tp_size: int, error: str -): - """Reject unsupported peers without registering agents or transfer state.""" - metadata = _agent_metadata([["a"]], [0xA000], [128]) - metadata.block_size = remote_block_size - worker = _layer_routing_worker([["a"]], {"a": 0}) - worker.block_size = local_block_size - worker.block_len_per_layer = [128] - worker.use_mla = False - worker.nixl_wrapper = MagicMock() - worker.kv_caches_base_addr = defaultdict(dict) - worker.dst_num_blocks = {} - worker.tp_mappings = {} - worker.transfer_topo = TransferTopology( - tp_rank=0, - tp_size=1, - block_size=local_block_size, - engine_id=worker.engine_id, - is_mla=False, - is_mamba=False, - total_num_kv_heads=8, - attn_backends=[], - ) - - with pytest.raises(NotImplementedError, match=error): - worker.add_remote_agent(metadata, remote_tp_size=remote_tp_size) - worker.nixl_wrapper.add_remote_agent.assert_not_called() - worker.nixl_wrapper.prep_xfer_dlist.assert_not_called() - assert not worker.tp_mappings - assert not worker.dst_num_blocks - assert not worker.kv_caches_base_addr - with pytest.raises(KeyError): - worker.transfer_topo.get_engine_info(metadata.engine_id) diff --git a/tests/v1/kv_connector/unit/test_remote_prefill_lifecycle.py b/tests/v1/kv_connector/unit/test_remote_prefill_lifecycle.py deleted file mode 100644 index 74de3c3a8c..0000000000 --- a/tests/v1/kv_connector/unit/test_remote_prefill_lifecycle.py +++ /dev/null @@ -1,977 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -import copy -from dataclasses import replace -from unittest.mock import patch - -import pytest - -from vllm.v1.metrics.stats import KV_FETCH_STAGES -from vllm.v1.outputs import ( - EMPTY_MODEL_RUNNER_OUTPUT, - KVConnectorOutput, - ModelRunnerOutput, -) -from vllm.v1.request import FinishReason, RequestStatus - -from .utils import ( - assert_scheduler_empty, - create_model_runner_output, - create_request, - create_scheduler, - create_vllm_config, - make_kv_cache_config, -) - -pytestmark = pytest.mark.cpu_test - - -def _num_waiting_requests(scheduler) -> int: - return len(scheduler.waiting) + len(scheduler.kv_holding_waiting) - - -def _kv_fetch_stages(scheduler) -> tuple[int, ...]: - """(waiting_to_start, in_progress, completed_waiting) request counts.""" - return tuple(scheduler._kv_fetch_counts[s] for s in KV_FETCH_STAGES) - - -def test_basic_lifecycle(): - """Test lifecycle of a remote prefill.""" - vllm_config = create_vllm_config() - scheduler = create_scheduler(vllm_config) - - # 2 Full Blocks and 1 Half Block. - BLOCK_SIZE = vllm_config.cache_config.block_size - NUM_EXTERNAL_FULL_BLOCKS = 2 - NUM_TOKENS = int(BLOCK_SIZE * (NUM_EXTERNAL_FULL_BLOCKS + 0.5)) - START_FREE_BLOCK_QUEUE_SIZE = ( - scheduler.kv_cache_manager.block_pool.free_block_queue.num_free_blocks - ) - - request = create_request( - request_id=1, - block_size=BLOCK_SIZE, - num_tokens=NUM_TOKENS, - do_remote_prefill=True, - ) - - scheduler.add_request(request) - request_id = request.request_id - assert _kv_fetch_stages(scheduler) == (1, 0, 0) - - # STEP (1): - # (1a): schedule() - scheduler_output = scheduler.schedule() - assert _kv_fetch_stages(scheduler) == (0, 1, 0) - - # Nothing running and empty scheduler output. - assert len(scheduler.running) == 0 - assert len(scheduler_output.scheduled_new_reqs) == 0 - assert scheduler_output.scheduled_cached_reqs.num_reqs == 0 - assert len(scheduler_output.num_scheduled_tokens) == 0 - assert scheduler_output.total_num_scheduled_tokens == 0 - - # Req waiting for KVs with no computed/scheduled toks ... - assert _num_waiting_requests(scheduler) == 1 - assert request in scheduler.kv_holding_waiting - assert request.status == RequestStatus.WAITING_FOR_REMOTE_KVS - assert request.num_computed_tokens == NUM_TOKENS - - # ... but should have (uncached) blocks allocated to it. - block_pool = scheduler.kv_cache_manager.block_pool - assert block_pool.free_block_queue.num_free_blocks < START_FREE_BLOCK_QUEUE_SIZE - assert len(block_pool.cached_block_hash_to_block) == 0 - blocks = scheduler.kv_cache_manager.coordinator.single_type_managers[ - 0 - ].req_to_blocks[request_id] - for block in blocks: - assert block._block_hash is None - - # (1b): forward() - model_runner_output = EMPTY_MODEL_RUNNER_OUTPUT - - # (1c): update_from_output() - engine_core_outputs = scheduler.update_from_output( - scheduler_output, model_runner_output - ) - assert not engine_core_outputs or not engine_core_outputs[0].outputs - - # STEP (2): - # (2a): schedule(): nothing happens! - scheduler_output = scheduler.schedule() - assert _num_waiting_requests(scheduler) == 1 - assert len(scheduler.running) == 0 - - # (2b): forward(): request finishes recv. - model_runner_output = copy.deepcopy(EMPTY_MODEL_RUNNER_OUTPUT) - model_runner_output.kv_connector_output = KVConnectorOutput( - finished_recving={request_id} - ) - - # (2c): update_from_output(): - engine_core_outputs = scheduler.update_from_output( - scheduler_output, model_runner_output - ) - assert _num_waiting_requests(scheduler) == 1 - assert request_id in scheduler.finished_recving_kv_req_ids - assert _kv_fetch_stages(scheduler) == (0, 0, 1) - - # STEP (3): - # (3a): schedule(): this should actually schedule. - scheduler_output = scheduler.schedule() - assert len(scheduler.running) == 1 - assert _kv_fetch_stages(scheduler) == (0, 0, 0) - - # Confirm the block are actually allocated. - num_hashed_blocks = 0 - blocks = scheduler.kv_cache_manager.coordinator.single_type_managers[ - 0 - ].req_to_blocks[request_id] - for block in blocks: - assert block.ref_cnt == 1 - num_hashed_blocks += 1 if block._block_hash is not None else 0 - assert num_hashed_blocks == NUM_EXTERNAL_FULL_BLOCKS - - # Confirm the rest of the prompt is scheduled in this step. - scheduled_req = scheduler_output.scheduled_new_reqs[0] - num_scheduled_tokens = scheduler_output.num_scheduled_tokens[request_id] - num_computed_tokens = scheduled_req.num_computed_tokens - total_prompt_tokens = len(scheduled_req.prompt_token_ids) - assert num_scheduled_tokens == total_prompt_tokens - num_computed_tokens - - # (3b): execute_model() - model_runner_output = create_model_runner_output([request]) - # (3c): update_from_output() - scheduler.update_from_output(scheduler_output, model_runner_output) - - # Step (4): Hit EOS. - scheduler_output = scheduler.schedule() - model_runner_output = create_model_runner_output([request], use_eos=True) - engine_core_outputs = scheduler.update_from_output( - scheduler_output, model_runner_output - ) - scheduler.schedule() - - outputs = engine_core_outputs[0].outputs - assert len(outputs) == 1 - output = outputs[0] - assert output.finish_reason == FinishReason.STOP - assert_scheduler_empty(scheduler) - - -def test_interleaved_lifecycle(): - """Test Remote Prefills Work Well With Other Requests.""" - vllm_config = create_vllm_config() - scheduler = create_scheduler(vllm_config) - - # 2 Full Blocks and 1 Half Block. - BLOCK_SIZE = vllm_config.cache_config.block_size - NUM_EXTERNAL_FULL_BLOCKS = 2 - NUM_TOKENS = int(BLOCK_SIZE * (NUM_EXTERNAL_FULL_BLOCKS + 0.5)) - - request_remote = create_request( - request_id=1, - block_size=BLOCK_SIZE, - num_tokens=NUM_TOKENS, - do_remote_prefill=True, - ) - request_local_a = create_request( - request_id=2, - block_size=BLOCK_SIZE, - num_tokens=NUM_TOKENS, - ) - request_local_b = create_request( - request_id=3, - block_size=BLOCK_SIZE, - num_tokens=NUM_TOKENS, - ) - - # STEP 1: Regular request is running. - scheduler.add_request(request_local_a) - scheduler_output = scheduler.schedule() - assert len(scheduler.running) == 1 - - model_runner_output = create_model_runner_output([request_local_a]) - scheduler.update_from_output(scheduler_output, model_runner_output) - - # STEP 2: Add a local and remote request. - scheduler.add_request(request_local_b) - scheduler.add_request(request_remote) - scheduler_output = scheduler.schedule() - assert len(scheduler.running) == 2 - assert _num_waiting_requests(scheduler) == 1 - assert len(scheduler_output.scheduled_new_reqs) == 1 - assert scheduler_output.scheduled_cached_reqs.num_reqs == 1 - - model_runner_output = create_model_runner_output([request_local_a, request_local_b]) - scheduler.update_from_output(scheduler_output, model_runner_output) - - # STEP 3: continue running, KVs not arrived yet. - scheduler_output = scheduler.schedule() - assert len(scheduler.running) == 2 - assert _num_waiting_requests(scheduler) == 1 - assert len(scheduler_output.scheduled_new_reqs) == 0 - assert scheduler_output.scheduled_cached_reqs.num_reqs == 2 - - model_runner_output = create_model_runner_output( - reqs=[request_local_a, request_local_b] - ) - scheduler.update_from_output(scheduler_output, model_runner_output) - assert len(scheduler.running) == 2 - assert _num_waiting_requests(scheduler) == 1 - assert len(scheduler_output.scheduled_new_reqs) == 0 - assert scheduler_output.scheduled_cached_reqs.num_reqs == 2 - - # STEP 4: KVs arrive. - scheduler_output = scheduler.schedule() - assert len(scheduler.running) == 2 - assert _num_waiting_requests(scheduler) == 1 - assert len(scheduler_output.scheduled_new_reqs) == 0 - assert scheduler_output.scheduled_cached_reqs.num_reqs == 2 - - model_runner_output = create_model_runner_output( - [request_local_a, request_local_b], finished_recving={request_remote.request_id} - ) - scheduler.update_from_output(scheduler_output, model_runner_output) - - # STEP 5: RECVed KVs are sent to ModelRunner. - scheduler_output = scheduler.schedule() - assert len(scheduler.running) == 3 - assert _num_waiting_requests(scheduler) == 0 - assert len(scheduler_output.scheduled_new_reqs) == 1 - assert scheduler_output.scheduled_cached_reqs.num_reqs == 2 - - model_runner_output = create_model_runner_output( - [request_local_a, request_local_b, request_remote] - ) - scheduler.update_from_output(scheduler_output, model_runner_output) - - # STEP 6: Hit EOS and free. - scheduler_output = scheduler.schedule() - model_runner_output = create_model_runner_output( - [request_local_a, request_local_b, request_remote], - use_eos=True, - ) - scheduler.update_from_output(scheduler_output, model_runner_output) - scheduler.schedule() - assert_scheduler_empty(scheduler) - - -def test_no_spurious_prefix_caching(): - """With P/D, blocks can be allocated but uncomputed for - multiple engine steps. This test confirms that we do - not accidentally have cache hits against uncomputed - blocks. - """ - vllm_config = create_vllm_config() - scheduler = create_scheduler(vllm_config) - - vllm_config = create_vllm_config() - scheduler = create_scheduler(vllm_config) - - # 2 and a half full external blocks. - BLOCK_SIZE = vllm_config.cache_config.block_size - NUM_EXTERNAL_FULL_BLOCKS = 2 - NUM_TOKENS = int(BLOCK_SIZE * (NUM_EXTERNAL_FULL_BLOCKS + 0.5)) - - # Both of these requests have prompts like [1,1,1,1,1, ...] - request_remote = create_request( - request_id=1, - block_size=BLOCK_SIZE, - num_tokens=NUM_TOKENS, - common_prefix_len=NUM_TOKENS, - do_remote_prefill=True, - ) - - request_local = create_request( - request_id=2, - block_size=BLOCK_SIZE, - num_tokens=NUM_TOKENS, - common_prefix_len=NUM_TOKENS, - do_remote_prefill=False, - ) - - # Schedule the remote prefill request. This should not - # cause any blocks to be cached. - scheduler.add_request(request_remote) - scheduler_output = scheduler.schedule() - scheduler.update_from_output(scheduler_output, EMPTY_MODEL_RUNNER_OUTPUT) - assert _num_waiting_requests(scheduler) == 1 - - # Schedule the local prefill request. This should - # cause blocks to be cached, but separately from - scheduler.add_request(request_local) - scheduler_output = scheduler.schedule() - assert len(scheduler.running) == 1 - assert _num_waiting_requests(scheduler) == 1 - - local_blocks = scheduler.kv_cache_manager.coordinator.single_type_managers[ - 0 - ].req_to_blocks[request_local.request_id] - remote_blocks = scheduler.kv_cache_manager.coordinator.single_type_managers[ - 0 - ].req_to_blocks[request_remote.request_id] - - # Local should have cached blocks (but not all due to preallocate). - num_hashed_blocks = 0 - for block in local_blocks: - assert block.ref_cnt == 1 - num_hashed_blocks += 1 if block._block_hash is not None else 0 - assert num_hashed_blocks > 0 - - # Remote blocks should not be cached. - for block in remote_blocks: - assert block.ref_cnt == 1 - assert block._block_hash is None - - -def test_full_block_prompt(): - """Test that we handle a prompt that is the full block size.""" - vllm_config = create_vllm_config() - scheduler = create_scheduler(vllm_config) - - # 2 Full Blocks and 1 Half Block. - BLOCK_SIZE = vllm_config.cache_config.block_size - NUM_EXTERNAL_FULL_BLOCKS = 2 - NUM_TOKENS = int(BLOCK_SIZE * NUM_EXTERNAL_FULL_BLOCKS) - - request = create_request( - request_id=1, - block_size=BLOCK_SIZE, - num_tokens=NUM_TOKENS, - do_remote_prefill=True, - ) - - scheduler.add_request(request) - request_id = request.request_id - - # STEP (1): Initialize a recv. - scheduler_output = scheduler.schedule() - # All blocks should be allocated. - num_blocks = len( - scheduler.kv_cache_manager.coordinator.single_type_managers[0].req_to_blocks[ - request_id - ] - ) - assert num_blocks == NUM_EXTERNAL_FULL_BLOCKS - model_runner_output = EMPTY_MODEL_RUNNER_OUTPUT - scheduler.update_from_output(scheduler_output, model_runner_output) - - # # STEP (2): Recv. - scheduler_output = scheduler.schedule() - model_runner_output = copy.deepcopy(EMPTY_MODEL_RUNNER_OUTPUT) - model_runner_output.kv_connector_output = KVConnectorOutput( - finished_recving={request_id} - ) - scheduler.update_from_output(scheduler_output, model_runner_output) - assert _num_waiting_requests(scheduler) == 1 - assert request_id in scheduler.finished_recving_kv_req_ids - - # # STEP (3): Run as usual. - scheduler_output = scheduler.schedule() - - # We need to recompute the final token of the prompt to generate - # the first new token, so we should not have a new block. - num_blocks = len( - scheduler.kv_cache_manager.coordinator.single_type_managers[0].req_to_blocks[ - request_id - ] - ) - assert num_blocks == NUM_EXTERNAL_FULL_BLOCKS - assert scheduler_output.scheduled_new_reqs[0].num_computed_tokens == NUM_TOKENS - 1 - assert scheduler_output.num_scheduled_tokens[request_id] == 1 - - model_runner_output = create_model_runner_output([request]) - scheduler.update_from_output(scheduler_output, model_runner_output) - - # # Step (4): Hit EOS. - scheduler_output = scheduler.schedule() - model_runner_output = create_model_runner_output([request], use_eos=True) - engine_core_outputs = scheduler.update_from_output( - scheduler_output, model_runner_output - ) - scheduler.schedule() - - outputs = engine_core_outputs[0].outputs - assert len(outputs) == 1 - output = outputs[0] - assert output.finish_reason == FinishReason.STOP - assert_scheduler_empty(scheduler) - - -def test_cannot_schedule_after_recv(): - """Test that we can handle no schedule after recv due to not - enough remaining KV blocks. - """ - # NOTE: the KVCacheManager will use 1 null block. - # So there are 5 total working blocks. - TOTAL_NUM_BLOCKS = 6 - vllm_config = create_vllm_config() - scheduler = create_scheduler(vllm_config, num_blocks=TOTAL_NUM_BLOCKS) - - # Prime the KVCache. - NUM_PROMPT_BLOCKS = 2 - BLOCK_SIZE = vllm_config.cache_config.block_size - # Prompt will use 2 blocks + 1 block after we schedule. - NUM_TOKENS_LOCAL = int(BLOCK_SIZE * NUM_PROMPT_BLOCKS) - NUM_TOKENS_REMOTE = int(BLOCK_SIZE * NUM_PROMPT_BLOCKS) - - request_normal = create_request( - request_id=1, block_size=BLOCK_SIZE, num_tokens=NUM_TOKENS_LOCAL - ) - request_remote = create_request( - request_id=2, - block_size=BLOCK_SIZE, - num_tokens=NUM_TOKENS_REMOTE, - do_remote_prefill=True, - ) - - # STEP 1: 3 blocks are in use (2 for prompt, 1 for decode). - scheduler.add_request(request_normal) - scheduler_output = scheduler.schedule() - model_runner_output = create_model_runner_output(reqs=[request_normal]) - scheduler.update_from_output(scheduler_output, model_runner_output) - assert len(scheduler.running) == 1 - assert _num_waiting_requests(scheduler) == 0 - - # Step 2: 5 blocks are in use (2 new for remote blocks). - scheduler.add_request(request_remote) - scheduler_output = scheduler.schedule() - model_runner_output = create_model_runner_output(reqs=[request_normal]) - scheduler.update_from_output(scheduler_output, model_runner_output) - assert len(scheduler.running) == 1 - assert _num_waiting_requests(scheduler) == 1 - - # Step 3: finish recving (5 blocks in use) - scheduler_output = scheduler.schedule() - model_runner_output = create_model_runner_output( - reqs=[request_normal], finished_recving={request_remote.request_id} - ) - scheduler.update_from_output(scheduler_output, model_runner_output) - assert len(scheduler.running) == 1 - assert _num_waiting_requests(scheduler) == 1 - - # Step 4: try to schedule, remote request is put to running list - # because the transfer is completed. - scheduler_output = scheduler.schedule() - model_runner_output = create_model_runner_output( - reqs=[request_normal, request_remote] - ) - scheduler.update_from_output(scheduler_output, model_runner_output) - assert len(scheduler.running) == 2 - assert _num_waiting_requests(scheduler) == 0 - - # Step 5: Remote request will be put back to waiting list - # because it needs new block to hold generated token. - scheduler_output = scheduler.schedule() - model_runner_output = create_model_runner_output(reqs=[request_normal]) - scheduler.update_from_output(scheduler_output, model_runner_output) - assert len(scheduler.running) == 1 - assert _num_waiting_requests(scheduler) == 1 - - # Step 6: finish the request, free it. - scheduler_output = scheduler.schedule() - model_runner_output = create_model_runner_output( - reqs=[request_normal], use_eos=True - ) - scheduler.update_from_output(scheduler_output, model_runner_output) - assert len(scheduler.running) == 0 - assert _num_waiting_requests(scheduler) == 1 - - # Step 7: now we can schedule (with 2 blocks computed), - # request is retrieved from preempted list. - scheduler_output = scheduler.schedule() - model_runner_output = create_model_runner_output(reqs=[request_remote]) - # V2 emits a resumed (previously preempted) request as a NewRequestData - # rather than a cached request. - if scheduler.use_v2_model_runner: - num_computed = scheduler_output.scheduled_new_reqs[0].num_computed_tokens - else: - cached = scheduler_output.scheduled_cached_reqs - num_computed = cached.num_computed_tokens[0] - assert num_computed == NUM_PROMPT_BLOCKS * BLOCK_SIZE - scheduler.update_from_output(scheduler_output, model_runner_output) - assert len(scheduler.running) == 1 - assert _num_waiting_requests(scheduler) == 0 - - # Step 8: free everything. - scheduler_output = scheduler.schedule() - model_runner_output = create_model_runner_output( - reqs=[request_remote], use_eos=True - ) - scheduler.update_from_output(scheduler_output, model_runner_output) - _ = scheduler.schedule() - assert_scheduler_empty(scheduler) - - -def test_cannot_recv(): - """Test that we can handle no schedule KV block transfer due to not - enough remaining KV blocks. - """ - # NOTE: the KVCacheManager will use 1 null block. - # So there are 5 total working blocks. - TOTAL_NUM_BLOCKS = 6 - vllm_config = create_vllm_config() - scheduler = create_scheduler(vllm_config, num_blocks=TOTAL_NUM_BLOCKS) - - # Prime the KVCache. - NUM_PROMPT_BLOCKS = 2 - BLOCK_SIZE = vllm_config.cache_config.block_size - # Prompt will use 2 blocks + 1 block after we schedule. - NUM_TOKENS_LOCAL = int(BLOCK_SIZE * NUM_PROMPT_BLOCKS) - NUM_TOKENS_REMOTE = int(BLOCK_SIZE * (NUM_PROMPT_BLOCKS + 0.5)) - - request_normal = create_request( - request_id=1, block_size=BLOCK_SIZE, num_tokens=NUM_TOKENS_LOCAL - ) - request_remote = create_request( - request_id=2, - block_size=BLOCK_SIZE, - num_tokens=NUM_TOKENS_REMOTE, - do_remote_prefill=True, - ) - - # STEP 1: 3 blocks are in use (2 for prompt, 1 for decode). - scheduler.add_request(request_normal) - scheduler_output = scheduler.schedule() - model_runner_output = create_model_runner_output(reqs=[request_normal]) - scheduler.update_from_output(scheduler_output, model_runner_output) - assert len(scheduler.running) == 1 - assert _num_waiting_requests(scheduler) == 0 - - # Step 2: 3 blocks are in use, - # need 3 new for remote blocks but only 2 are available. - scheduler.add_request(request_remote) - scheduler_output = scheduler.schedule() - model_runner_output = create_model_runner_output(reqs=[request_normal]) - scheduler.update_from_output(scheduler_output, model_runner_output) - assert len(scheduler.running) == 1 - assert _num_waiting_requests(scheduler) == 1 - # Should not have KV transfer in progress. - assert request_remote.status != RequestStatus.WAITING_FOR_REMOTE_KVS - - # Step 3: finish the request, free it. - scheduler_output = scheduler.schedule() - model_runner_output = create_model_runner_output( - reqs=[request_normal], use_eos=True - ) - scheduler.update_from_output(scheduler_output, model_runner_output) - assert len(scheduler.running) == 0 - assert _num_waiting_requests(scheduler) == 1 - - # Step 4: now we can initiate KV transfer (with 2 blocks computed). - scheduler_output = scheduler.schedule() - model_runner_output = create_model_runner_output(reqs=[]) - scheduler.update_from_output(scheduler_output, model_runner_output) - assert len(scheduler.running) == 0 - assert _num_waiting_requests(scheduler) == 1 - assert request_remote.status == RequestStatus.WAITING_FOR_REMOTE_KVS - - # Step 5: finish recving (5 blocks in use) - scheduler_output = scheduler.schedule() - model_runner_output = create_model_runner_output( - reqs=[], finished_recving={request_remote.request_id} - ) - scheduler.update_from_output(scheduler_output, model_runner_output) - assert len(scheduler.running) == 0 - assert _num_waiting_requests(scheduler) == 1 - - # Step 6: schedule remote request - scheduler_output = scheduler.schedule() - model_runner_output = create_model_runner_output(reqs=[request_remote]) - scheduler.update_from_output(scheduler_output, model_runner_output) - assert len(scheduler.running) == 1 - assert _num_waiting_requests(scheduler) == 0 - - # Step 7: free everything. - scheduler_output = scheduler.schedule() - model_runner_output = create_model_runner_output( - reqs=[request_remote], use_eos=True - ) - scheduler.update_from_output(scheduler_output, model_runner_output) - _ = scheduler.schedule() - assert_scheduler_empty(scheduler) - - -def test_kv_fetch_stages_cleared_on_abort(): - vllm_config = create_vllm_config() - scheduler = create_scheduler(vllm_config) - BLOCK_SIZE = vllm_config.cache_config.block_size - request = create_request( - request_id=1, - block_size=BLOCK_SIZE, - num_tokens=int(BLOCK_SIZE * 2.5), - do_remote_prefill=True, - ) - scheduler.add_request(request) - scheduler.schedule() - assert _kv_fetch_stages(scheduler) == (0, 1, 0) - - scheduler.finish_requests(request.request_id, RequestStatus.FINISHED_ABORTED) - assert _kv_fetch_stages(scheduler) == (0, 0, 0) - - -@patch( - "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_scheduler.current_platform" -) -def test_p_side_chunked_prefill_mamba(mock_platform): - """P-side integration: Mamba N-1 truncation + chunked prefill completes. - - A 64-token P-side request is truncated to 63 by the N-1 fix, then - chunked into two prefill steps (32 + 31) and finishes with - LENGTH_CAPPED because max_tokens is set to 1. - """ - mock_platform.device_type = "cpu" - - BATCH_SIZE = 32 - NUM_TOKENS = 64 - BLOCK_SIZE = 16 - - vllm_config = create_vllm_config( - max_num_batched_tokens=BATCH_SIZE, - block_size=BLOCK_SIZE, - ) - vllm_config.scheduler_config.disable_hybrid_kv_cache_manager = False - # mamba_cache_mode "none" only runs without prefix caching. - vllm_config.cache_config.enable_prefix_caching = False - - kv_cache_config = make_kv_cache_config( - block_size=BLOCK_SIZE, - mamba_enabled=True, - num_blocks=10000, - ) - - scheduler = create_scheduler(vllm_config, kv_cache_config=kv_cache_config) - - request = create_request( - num_tokens=NUM_TOKENS, - do_remote_decode=True, - block_size=BLOCK_SIZE, - ) - request.max_tokens = 128 - scheduler.add_request(request) - request_id = request.request_id - - # ── Step 1: first chunk ── - scheduler_output = scheduler.schedule() - - assert len(request.prompt_token_ids) == NUM_TOKENS - 1 - assert request.max_tokens == 1 - assert scheduler_output.num_scheduled_tokens[request_id] == BATCH_SIZE - assert request.num_computed_tokens == BATCH_SIZE - - # Model returns no tokens for intermediate prefill chunk - intermediate_output = ModelRunnerOutput( - req_ids=[request.request_id], - req_id_to_index={request.request_id: 0}, - sampled_token_ids=[[]], - ) - scheduler.update_from_output(scheduler_output, intermediate_output) - - # ── Step 2: remaining chunk ── - scheduler_output = scheduler.schedule() - - remaining = NUM_TOKENS - 1 - BATCH_SIZE # 31 - assert scheduler_output.num_scheduled_tokens[request_id] == remaining - assert request.num_computed_tokens == NUM_TOKENS - 1 - - # Prefill complete: model generates 1 decode token - final_output = create_model_runner_output([request]) - engine_core_outputs = scheduler.update_from_output(scheduler_output, final_output) - - # max_tokens=1 → request finishes with LENGTH - outputs = engine_core_outputs[0].outputs - assert len(outputs) == 1 - assert outputs[0].finish_reason == FinishReason.LENGTH - - -def test_async_load_reserves_blocks_for_inflight(): - """A second async KV-connector load is not admitted if its initial - allocation would consume blocks reserved for an already in-flight sequence. - - req_a gets a 1-block prefix (full sequence = 4 blocks), reserving 3 more. - req_b would need a 4-block initial allocation, but only - (free - req_a's 3-block reservation) = 3 blocks are available to it, so it is - held back in WAITING (holding no blocks) rather than wedging req_a. - """ - vllm_config = create_vllm_config() - BLOCK_SIZE = vllm_config.cache_config.block_size - scheduler = create_scheduler(vllm_config, num_blocks=8) # usable = 7 - - req_a = create_request( - request_id=1, - block_size=BLOCK_SIZE, - num_tokens=BLOCK_SIZE * 4, - do_remote_prefill=True, - num_remote_blocks=1, - ) - req_b = create_request( - request_id=2, - block_size=BLOCK_SIZE, - num_tokens=BLOCK_SIZE * 5, - do_remote_prefill=True, - num_remote_blocks=1, - ) - scheduler.add_request(req_a) - scheduler.add_request(req_b) - - # Partial external matches: req_a loads 1 block, req_b loads 4 blocks. - with patch.object( - scheduler.connector, - "get_num_new_matched_tokens", - side_effect=[(BLOCK_SIZE, True), (BLOCK_SIZE * 4, True)], - ): - scheduler.schedule() - - assert req_a.status == RequestStatus.WAITING_FOR_REMOTE_KVS - assert req_b.status == RequestStatus.WAITING - - req_to_blocks = scheduler.kv_cache_manager.coordinator.single_type_managers[ - 0 - ].req_to_blocks - assert req_a.request_id in req_to_blocks - assert req_b.request_id not in req_to_blocks - - -def test_async_loads_both_admitted_when_pool_fits(): - """Sanity: with a pool large enough, the reservation gate admits both async - loads (it is not over-conservative).""" - vllm_config = create_vllm_config() - BLOCK_SIZE = vllm_config.cache_config.block_size - scheduler = create_scheduler(vllm_config, num_blocks=64) - - reqs = [ - create_request( - request_id=i, - block_size=BLOCK_SIZE, - num_tokens=BLOCK_SIZE * 5, - do_remote_prefill=True, - num_remote_blocks=1, - ) - for i in (1, 2) - ] - for req in reqs: - scheduler.add_request(req) - - with patch.object( - scheduler.connector, - "get_num_new_matched_tokens", - side_effect=[(BLOCK_SIZE, True), (BLOCK_SIZE, True)], - ): - scheduler.schedule() - - for req in reqs: - assert req.status == RequestStatus.WAITING_FOR_REMOTE_KVS - - -@pytest.mark.parametrize( - ("swa_enabled", "block_size", "num_spec_tokens", "num_blocks"), - [(False, 16, 3, 8), (True, 16, 3, 17), (True, 64, 31, 40)], - ids=["single-group", "multiple-groups", "different-block-sizes"], -) -def test_async_load_reserves_blocks_for_promotion_margin( - swa_enabled, block_size, num_spec_tokens, num_blocks -): - """Async loads must leave enough blocks for promotion in every cache group. - - Otherwise two loads can fill the pool and prevent the head from being - promoted, with nothing running to free blocks and break the deadlock. - """ - vllm_config = create_vllm_config( - block_size=block_size, - num_speculative_tokens=num_spec_tokens, - disable_hybrid_kv_cache_manager=False, - ) - hash_block_size = 16 - kv_cache_config = make_kv_cache_config( - hash_block_size, swa_enabled=swa_enabled, sw_size=4096, num_blocks=num_blocks - ) - full_group = kv_cache_config.kv_cache_groups[0] - full_group.kv_cache_spec = replace(full_group.kv_cache_spec, block_size=block_size) - if swa_enabled: - sliding_group = kv_cache_config.kv_cache_groups[-1] - sliding_group.kv_cache_spec = replace( - sliding_group.kv_cache_spec, - page_size_padded=full_group.kv_cache_spec.page_size_bytes, - ) - scheduler = create_scheduler( - vllm_config, - num_blocks=num_blocks, - kv_cache_config=kv_cache_config, - hash_block_size=hash_block_size, - ) - - req_a = create_request( - request_id=1, - block_size=hash_block_size, - num_tokens=block_size * 4, - do_remote_prefill=True, - max_tokens=1, - ) - req_b = create_request( - request_id=2, - block_size=hash_block_size, - num_tokens=block_size * 3, - do_remote_prefill=True, - max_tokens=1, - ) - scheduler.add_request(req_a) - scheduler.add_request(req_b) - - # Both get a full external hit, so each would hold its whole prompt. - with patch.object( - scheduler.connector, - "get_num_new_matched_tokens", - side_effect=[(block_size * 4, True), (block_size * 3, True)], - ): - scheduler.schedule() - - assert req_a.status == RequestStatus.WAITING_FOR_REMOTE_KVS - assert req_b.status == RequestStatus.WAITING - assert not any(scheduler.kv_cache_manager.get_block_ids(req_b.request_id)) - - # req_a's load lands: it must be promotable and actually get scheduled. - scheduler.update_from_output( - scheduler.schedule(), - create_model_runner_output([], finished_recving={req_a.request_id}), - ) - scheduler_output = scheduler.schedule() - assert req_a.status == RequestStatus.RUNNING - assert scheduler_output.num_scheduled_tokens[req_a.request_id] > 0 - - -@pytest.mark.parametrize( - ("mamba_cache_mode", "mamba_block_size", "num_blocks"), - [("none", 128, 18), ("align", 16, 18)], -) -def test_async_load_does_not_double_reserve_mamba_spec_blocks( - mamba_cache_mode, mamba_block_size, num_blocks -): - """Two hybrid loads fit when Mamba's speculative states are counted once.""" - block_size = 16 - num_spec_tokens = 3 - vllm_config = create_vllm_config( - block_size=mamba_block_size, - max_model_len=128, - num_speculative_tokens=num_spec_tokens, - disable_hybrid_kv_cache_manager=False, - ) - vllm_config.cache_config.mamba_cache_mode = mamba_cache_mode - vllm_config.cache_config.enable_prefix_caching = mamba_cache_mode == "align" - kv_cache_config = make_kv_cache_config( - block_size, - mamba_enabled=True, - mamba_cache_mode=mamba_cache_mode, - num_blocks=num_blocks, - ) - mamba_group = kv_cache_config.kv_cache_groups[-1] - mamba_group.kv_cache_spec = replace( - mamba_group.kv_cache_spec, - block_size=mamba_block_size, - num_speculative_blocks=num_spec_tokens, - ) - scheduler = create_scheduler( - vllm_config, - num_blocks=num_blocks, - kv_cache_config=kv_cache_config, - hash_block_size=block_size, - ) - reqs = [ - create_request( - request_id=i, - block_size=block_size, - num_tokens=num_tokens, - do_remote_prefill=True, - max_tokens=1, - ) - for i, num_tokens in enumerate((64, 48), 1) - ] - for req in reqs: - scheduler.add_request(req) - - with patch.object( - scheduler.connector, - "get_num_new_matched_tokens", - side_effect=[(req.num_tokens, True) for req in reqs], - ): - scheduler.schedule() - - assert all(req.status == RequestStatus.WAITING_FOR_REMOTE_KVS for req in reqs) - scheduler.update_from_output( - scheduler.schedule(), - create_model_runner_output( - [], finished_recving={req.request_id for req in reqs} - ), - ) - output = scheduler.schedule() - for req in reqs: - assert req.status == RequestStatus.RUNNING - assert output.num_scheduled_tokens[req.request_id] > 0 - - -def test_deferred_lookup_does_not_block_parked_load(): - """A request with a deferred connector lookup must not strand an async - load queued behind it. - - req_a's lookup is deferred (connector returns None), so it is parked in - waiting holding no blocks; req_b is then admitted as an async - load holding most of the pool. Once req_a's lookup resolves it cannot - allocate, and the waiting scan stops at it every step. The parked load - must still be promotable: it lives in the KV-holding queue - that is drained first, so it runs, finishes, and frees its blocks for - req_a. - """ - vllm_config = create_vllm_config() - BLOCK_SIZE = vllm_config.cache_config.block_size - scheduler = create_scheduler(vllm_config, num_blocks=8) # usable = 7 - - req_a = create_request( - request_id=1, block_size=BLOCK_SIZE, num_tokens=BLOCK_SIZE * 4, max_tokens=1 - ) - req_b = create_request( - request_id=2, - block_size=BLOCK_SIZE, - num_tokens=BLOCK_SIZE * 5, - do_remote_prefill=True, - max_tokens=1, - ) - scheduler.add_request(req_a) - scheduler.add_request(req_b) - - deferrals = [(None, False)] - - def lookup(request, num_computed_tokens): - if request is req_b: - return BLOCK_SIZE * 5, True - return deferrals.pop() if deferrals else (0, False) - - with patch.object( - scheduler.connector, "get_num_new_matched_tokens", side_effect=lookup - ): - # Step 1: req_a's lookup is deferred; req_b is admitted as an async - # load holding 5 of 7 usable blocks. - scheduler.schedule() - assert req_a.status == RequestStatus.WAITING - assert req_b.status == RequestStatus.WAITING_FOR_REMOTE_KVS - - # req_a's lookup resolves (no hit) while req_b's transfer is in - # flight; req_a needs 4 blocks but only 2 are free, so the waiting - # scan stops at it. req_b's transfer then finishes. - scheduler.update_from_output( - scheduler.schedule(), - create_model_runner_output([], finished_recving={req_b.request_id}), - ) - - # req_b must still be reached, promoted, and scheduled despite the - # scan stopping at req_a. - scheduler_output = scheduler.schedule() - assert req_b.status == RequestStatus.RUNNING - assert scheduler_output.num_scheduled_tokens[req_b.request_id] > 0 - - # req_b finishes and frees its blocks; now req_a fits and runs. - scheduler.update_from_output( - scheduler_output, create_model_runner_output([req_b]) - ) - scheduler_output = scheduler.schedule() - assert req_a.status == RequestStatus.RUNNING - assert scheduler_output.num_scheduled_tokens[req_a.request_id] > 0 diff --git a/tests/v1/kv_connector/unit/test_simple_cpu_offload_connector.py b/tests/v1/kv_connector/unit/test_simple_cpu_offload_connector.py deleted file mode 100644 index af4c7858fc..0000000000 --- a/tests/v1/kv_connector/unit/test_simple_cpu_offload_connector.py +++ /dev/null @@ -1,149 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Unit tests for SimpleCPUOffloadConnector-level behavior.""" - -from __future__ import annotations - -import pytest - -from tests.v1.kv_connector.unit.utils import create_vllm_config -from tests.v1.simple_kv_offload.test_scheduler import ( - _BYTES_PER_BLOCK, - BLOCK_SIZE, - DTYPE, - HEAD_SIZE, - NUM_KV_HEADS, -) -from vllm.distributed.kv_transfer.kv_connector.v1.base import KVConnectorRole -from vllm.distributed.kv_transfer.kv_connector.v1.simple_cpu_offload_connector import ( - SimpleCPUOffloadConnector, -) -from vllm.v1.kv_cache_interface import ( - CircularBufferSpec, - FullAttentionSpec, - KVCacheConfig, - KVCacheGroupSpec, - KVCacheTensor, -) - - -def _make_kv_cache_config(num_blocks: int = 16) -> KVCacheConfig: - fa_layers = ["layer0"] - spec = FullAttentionSpec( - block_size=BLOCK_SIZE, - num_kv_heads=NUM_KV_HEADS, - head_size=HEAD_SIZE, - dtype=DTYPE, - ) - groups = [KVCacheGroupSpec(fa_layers, spec)] - size = _BYTES_PER_BLOCK * num_blocks - tensors = [ - KVCacheTensor( - size=size, - layers=fa_layers, - layer_stride=size, - block_stride=_BYTES_PER_BLOCK, - ) - ] - return KVCacheConfig( - num_blocks=num_blocks, - kv_cache_tensors=tensors, - kv_cache_groups=groups, - ) - - -def _make_connector( - extra_config: dict | None = None, - num_gpu_blocks: int = 16, - cpu_bytes: int = _BYTES_PER_BLOCK * 8, -) -> SimpleCPUOffloadConnector: - kv_cache_config = _make_kv_cache_config(num_gpu_blocks) - config_dict = { - "cpu_bytes_to_use": cpu_bytes, - } - if extra_config: - config_dict.update(extra_config) - vllm_config = create_vllm_config( - kv_connector="SimpleCPUOffloadConnector", - kv_role="kv_both", - kv_connector_extra_config=config_dict, - ) - connector = SimpleCPUOffloadConnector( - vllm_config=vllm_config, - role=KVConnectorRole.SCHEDULER, - kv_cache_config=kv_cache_config, - ) - return connector - - -def test_disk_mode_rejects_non_positive_capacity() -> None: - """Disk mode with disk_capacity_bytes <= 0 raises ValueError. - - Without this guard, the worker operates in disk mode but the scheduler - classifies events as MEDIUM_CPU instead of MEDIUM_STORAGE, silently - mislabeling events. - """ - with pytest.raises(ValueError, match="disk_capacity_bytes > 0"): - _make_connector( - extra_config={ - "kv_offload_backend": "disk", - "disk_path": "/tmp/fake", - "disk_capacity_bytes": 0, - } - ) - - -def _make_ring_kv_cache_config(num_blocks: int = 16) -> KVCacheConfig: - """A paged full-attention group beside a per-request ring group.""" - full = FullAttentionSpec( - block_size=BLOCK_SIZE, - num_kv_heads=NUM_KV_HEADS, - head_size=HEAD_SIZE, - dtype=DTYPE, - ) - ring = CircularBufferSpec( - block_size=6, num_kv_heads=1, head_size=32, head_size_v=0, dtype=DTYPE - ) - return KVCacheConfig( - num_blocks=num_blocks, - kv_cache_tensors=[ - KVCacheTensor( - size=_BYTES_PER_BLOCK * num_blocks, - layers=["attn"], - layer_stride=_BYTES_PER_BLOCK * num_blocks, - block_stride=_BYTES_PER_BLOCK, - ), - KVCacheTensor( - size=ring.page_size_bytes * num_blocks, - layers=["ring"], - layer_stride=ring.page_size_bytes * num_blocks, - block_stride=ring.page_size_bytes, - ), - ], - kv_cache_groups=[ - KVCacheGroupSpec(["attn"], full), - KVCacheGroupSpec(["ring"], ring), - ], - ) - - -def test_ring_scratch_group_is_accepted_and_never_offloaded() -> None: - """The ring group's capacity is not a token granularity, it holds no - hashed blocks, and it never participates in CPU hits.""" - vllm_config = create_vllm_config( - kv_connector="SimpleCPUOffloadConnector", - kv_role="kv_both", - kv_connector_extra_config={"cpu_bytes_to_use": _BYTES_PER_BLOCK * 8}, - ) - connector = SimpleCPUOffloadConnector( - vllm_config=vllm_config, - role=KVConnectorRole.SCHEDULER, - kv_cache_config=_make_ring_kv_cache_config(), - ) - scheduler = connector.scheduler_manager - assert scheduler.fa_gidx == 0 - cpu_groups = scheduler.cpu_kv_cache_config.kv_cache_groups - assert [g.kv_cache_spec.prefix_cacheable for g in cpu_groups] == [True, False] - # Only the paged group takes part in CPU-side hit lookup. - attention_groups = scheduler.cpu_coordinator.attention_groups - assert [g.group_ids for g in attention_groups] == [[0]] diff --git a/tests/v1/kv_connector/unit/test_simple_cpu_offload_metrics.py b/tests/v1/kv_connector/unit/test_simple_cpu_offload_metrics.py deleted file mode 100644 index 16358dec7e..0000000000 --- a/tests/v1/kv_connector/unit/test_simple_cpu_offload_metrics.py +++ /dev/null @@ -1,466 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Unit tests for SimpleCPUOffloadConnector metrics.""" - -from __future__ import annotations - -import dataclasses -from types import SimpleNamespace -from typing import Any - -import pytest -from prometheus_client import Counter, Gauge, Histogram - -from tests.v1.kv_connector.unit.test_simple_cpu_offload_connector import ( - _make_connector, -) -from tests.v1.simple_kv_offload.test_scheduler import ( - make_request, - make_scheduler, - make_scheduler_output, -) -from vllm.distributed.kv_transfer.kv_connector.v1.simple_cpu_offload_connector import ( - SimpleCPUOffloadConnector, -) -from vllm.v1.outputs import KVConnectorOutput -from vllm.v1.simple_kv_offload.manager import ( - BoundaryStoreStats, - LoadRequestState, - TransferMeta, -) -from vllm.v1.simple_kv_offload.metrics import ( - INFO_LABELS, - OUTCOME_TO_FIELD, - MetricName, - SimpleCPUOffloadStats, - _MetricType, - _StatsKey, -) - -SAVE_OUTCOMES = MetricName.SAVE_OUTCOMES -LOAD_BLOCKS = MetricName.LOAD_BLOCKS -USED_BLOCKS = MetricName.USED_BLOCKS -PENDING_STORE_BLOCKS = MetricName.PENDING_STORE_BLOCKS -INFO = MetricName.INFO - - -class _FakeMetric: - def __init__(self, **kwargs: Any): - self.kwargs = kwargs - self.children: list[_FakeMetric] = [] - self.observed: list[int | float] = [] - self.increments: list[int | float] = [] - self.set_values: list[int | float] = [] - self.labelvalues: tuple[object, ...] = () - - def labels(self, *labelvalues: object) -> _FakeMetric: - child = _FakeMetric(**self.kwargs) - child.labelvalues = labelvalues - self.children.append(child) - return child - - def observe(self, value: int | float) -> None: - self.observed.append(value) - - def inc(self, value: int | float) -> None: - self.increments.append(value) - - def set(self, value: int | float) -> None: - self.set_values.append(value) - - -def _fake_metric_types() -> dict[type, type]: - return {Gauge: _FakeMetric, Counter: _FakeMetric, Histogram: _FakeMetric} - - -def _make_prom_metrics(per_engine: dict[int, list[object]] | None = None) -> Any: - from vllm.v1.simple_kv_offload.metrics import SimpleCPUOffloadPromMetrics - - return SimpleCPUOffloadPromMetrics( - vllm_config=SimpleNamespace(kv_transfer_config=None), - metric_types=_fake_metric_types(), - labelnames=["model_name", "engine"], - per_engine_labelvalues=per_engine or {0: ["model", "0"]}, - ) - - -def _stats_payload(types: dict, data: dict) -> dict[str, Any]: - return {_StatsKey.TYPES: types, _StatsKey.DATA: data} - - -def _values(stats: SimpleCPUOffloadStats, name: str) -> dict: - return stats.data[_StatsKey.DATA][name] - - -# --------------------------------------------------------------------------- -# Stats class -# --------------------------------------------------------------------------- - - -def test_build_kv_connector_stats_none_and_empty() -> None: - stats = SimpleCPUOffloadConnector.build_kv_connector_stats(data=None) - assert isinstance(stats, SimpleCPUOffloadStats) - assert stats.is_empty() - - stats = SimpleCPUOffloadConnector.build_kv_connector_stats(data={}) - assert isinstance(stats, SimpleCPUOffloadStats) - assert stats.is_empty() - - -def test_stats_round_trip_through_serialized_dict() -> None: - stats = SimpleCPUOffloadStats() - stats.increase_counter(SAVE_OUTCOMES, 3, ("stored",)) - stats.increase_counter(LOAD_BLOCKS, 4) - stats.set_gauge(USED_BLOCKS, 7) - stats.set_gauge(INFO, 1, ("cpu", "false", "false", "64")) - - rebuilt = SimpleCPUOffloadConnector.build_kv_connector_stats(data=stats.to_dict()) - assert isinstance(rebuilt, SimpleCPUOffloadStats) - assert _values(rebuilt, SAVE_OUTCOMES) == {("stored",): 3} - assert _values(rebuilt, LOAD_BLOCKS) == {(): 4} - assert _values(rebuilt, USED_BLOCKS) == {(): 7} - assert _values(rebuilt, INFO) == {("cpu", "false", "false", "64"): 1} - - -def test_stats_aggregate_sums_counters_and_keeps_latest_gauge() -> None: - stats1 = SimpleCPUOffloadStats() - stats1.increase_counter(SAVE_OUTCOMES, 3, ("stored",)) - stats1.increase_counter(SAVE_OUTCOMES, 1, ("dropped_cpu_full",)) - stats1.set_gauge(USED_BLOCKS, 7) - - stats2 = SimpleCPUOffloadStats() - stats2.increase_counter(SAVE_OUTCOMES, 4, ("stored",)) - stats2.set_gauge(USED_BLOCKS, 2) - - result = stats1.aggregate(stats2) - assert result is stats1 - outcomes = _values(stats1, SAVE_OUTCOMES) - assert outcomes == {("stored",): 7, ("dropped_cpu_full",): 1} - assert _values(stats1, USED_BLOCKS) == {(): 2} - - -def test_stats_aggregate_merges_types_from_other() -> None: - stats1 = SimpleCPUOffloadStats() - stats1.set_gauge(USED_BLOCKS, 1) - - stats2 = SimpleCPUOffloadStats() - stats2.increase_counter(LOAD_BLOCKS, 2) - - stats1.aggregate(stats2) - assert stats1.data[_StatsKey.TYPES][LOAD_BLOCKS] == _MetricType.COUNTER - assert _values(stats1, LOAD_BLOCKS) == {(): 2} - - -def test_stats_reduce_and_reset() -> None: - stats = SimpleCPUOffloadStats() - stats.increase_counter(SAVE_OUTCOMES, 3, ("stored",)) - stats.set_gauge(PENDING_STORE_BLOCKS, 4) - stats.increase_counter(LOAD_BLOCKS, 9) - stats.set_gauge(INFO, 1, ("cpu", "false", "false", "64")) - - reduced = stats.reduce() - assert reduced[f"{SAVE_OUTCOMES}:{('stored',)}"] == 3 - assert reduced[PENDING_STORE_BLOCKS] == 4 - assert reduced[LOAD_BLOCKS] == 9 - assert not any(key.startswith(INFO) for key in reduced) - - stats.reset() - assert stats.is_empty() - - -def test_outcome_labels_match_boundary_store_stats_fields() -> None: - """Every outcome label maps 1:1 to a BoundaryStoreStats counter field.""" - boundary_fields = { - f.name for f in dataclasses.fields(BoundaryStoreStats) if f.name != "published" - } - assert set(OUTCOME_TO_FIELD.values()) == boundary_fields - assert len(set(OUTCOME_TO_FIELD.values())) == len(OUTCOME_TO_FIELD) - - -# --------------------------------------------------------------------------- -# Prom metrics class -# --------------------------------------------------------------------------- - - -def test_prom_metrics_registers_tier1_metrics() -> None: - prom = _make_prom_metrics() - defs = prom._defs - assert set(defs) == { - SAVE_OUTCOMES, - LOAD_BLOCKS, - USED_BLOCKS, - PENDING_STORE_BLOCKS, - INFO, - } - # The prom class registers literal names (required by the docs - # generator); they must stay in sync with the emission-side constants. - assert {m.kwargs["name"] for m in defs.values()} == { - SAVE_OUTCOMES, - LOAD_BLOCKS, - USED_BLOCKS, - PENDING_STORE_BLOCKS, - INFO, - } - for metric in defs.values(): - assert metric.kwargs["documentation"] - assert metric.kwargs["labelnames"][:2] == ["model_name", "engine"] - - assert defs[SAVE_OUTCOMES].kwargs["labelnames"] == [ - "model_name", - "engine", - "outcome", - ] - assert defs[LOAD_BLOCKS].kwargs["labelnames"] == ["model_name", "engine"] - assert defs[INFO].kwargs["labelnames"] == [ - "model_name", - "engine", - *INFO_LABELS, - ] - assert defs[USED_BLOCKS].kwargs["labelnames"] == ["model_name", "engine"] - - # mostrecent: multiprocess scrapes expose one series per label set, not - # one pid-tagged series per writer process. - for gauge in (USED_BLOCKS, PENDING_STORE_BLOCKS, INFO): - assert defs[gauge].kwargs["multiprocess_mode"] == "mostrecent" - - -def test_prom_metrics_observe_routes_each_series() -> None: - prom = _make_prom_metrics() - prom.observe( - _stats_payload( - types={ - SAVE_OUTCOMES: _MetricType.COUNTER, - LOAD_BLOCKS: _MetricType.COUNTER, - USED_BLOCKS: _MetricType.GAUGE, - PENDING_STORE_BLOCKS: _MetricType.GAUGE, - INFO: _MetricType.GAUGE, - }, - data={ - SAVE_OUTCOMES: {("stored",): 3}, - LOAD_BLOCKS: {(): 4}, - USED_BLOCKS: {(): 5}, - PENDING_STORE_BLOCKS: {(): 1}, - INFO: {("cpu", "false", "false", "64"): 1}, - }, - ) - ) - - assert prom._metrics[(0, SAVE_OUTCOMES, ("stored",))].increments == [3] - assert prom._metrics[(0, LOAD_BLOCKS, ())].increments == [4] - assert prom._metrics[(0, USED_BLOCKS, ())].set_values == [5] - assert prom._metrics[(0, PENDING_STORE_BLOCKS, ())].set_values == [1] - assert prom._metrics[(0, INFO, ("cpu", "false", "false", "64"))].set_values == [1] - assert prom._metrics[(0, SAVE_OUTCOMES, ("stored",))].labelvalues == ( - "model", - "0", - "stored", - ) - assert prom._metrics[(0, INFO, ("cpu", "false", "false", "64"))].labelvalues == ( - "model", - "0", - "cpu", - "false", - "false", - "64", - ) - - -def test_prom_metrics_reuses_bound_children() -> None: - prom = _make_prom_metrics() - payload = _stats_payload( - types={SAVE_OUTCOMES: _MetricType.COUNTER}, - data={SAVE_OUTCOMES: {("stored",): 1}}, - ) - prom.observe(payload) - prom.observe(payload) - - child = prom._metrics[(0, SAVE_OUTCOMES, ("stored",))] - assert child.increments == [1, 1] - assert len(prom._defs[SAVE_OUTCOMES].children) == 1 - - -def test_prom_metrics_rejects_unknown_metric() -> None: - prom = _make_prom_metrics() - with pytest.raises(AssertionError, match="Unknown"): - prom.observe( - _stats_payload( - types={"vllm:not_a_metric": _MetricType.COUNTER}, - data={"vllm:not_a_metric": {(): 1}}, - ) - ) - - -def test_prom_metrics_rejects_wrong_label_count() -> None: - prom = _make_prom_metrics() - with pytest.raises(AssertionError, match="labels"): - prom.observe( - _stats_payload( - types={SAVE_OUTCOMES: _MetricType.COUNTER}, - data={SAVE_OUTCOMES: {("stored", "extra"): 1}}, - ) - ) - - -def test_prom_metrics_routes_per_engine() -> None: - prom = _make_prom_metrics(per_engine={0: ["model", "0"], 1: ["model", "1"]}) - prom.observe( - _stats_payload( - types={USED_BLOCKS: _MetricType.GAUGE}, data={USED_BLOCKS: {(): 3}} - ), - engine_idx=1, - ) - assert (0, USED_BLOCKS, ()) not in prom._metrics - assert prom._metrics[(1, USED_BLOCKS, ())].set_values == [3] - - -def test_connector_build_prom_metrics() -> None: - from vllm.v1.simple_kv_offload.metrics import SimpleCPUOffloadPromMetrics - - prom = SimpleCPUOffloadConnector.build_prom_metrics( - vllm_config=SimpleNamespace(kv_transfer_config=None), - metric_types=_fake_metric_types(), - labelnames=["model_name", "engine"], - per_engine_labelvalues={0: ["model", "0"]}, - ) - assert isinstance(prom, SimpleCPUOffloadPromMetrics) - - -# --------------------------------------------------------------------------- -# Manager: interval stats -# --------------------------------------------------------------------------- - - -def test_manager_get_stats_reports_outcome_deltas_per_interval() -> None: - fixture = make_scheduler() - sched = fixture.scheduler - - sched.boundary_store_stats.stored += 3 - sched.boundary_store_stats.dropped_cpu_full += 2 - sched.boundary_store_stats.skipped_in_flight += 1 - - stats = sched.get_stats() - assert _values(stats, SAVE_OUTCOMES) == { - ("stored",): 3, - ("dropped_cpu_full",): 2, - ("skipped_in_flight",): 1, - } - - # Deltas are per-interval: a second drain reports no new outcomes. - stats2 = sched.get_stats() - assert SAVE_OUTCOMES not in stats2.data[_StatsKey.DATA] - - -def test_manager_reset_preserves_undrained_outcome_deltas() -> None: - fixture = make_scheduler() - sched = fixture.scheduler - - sched.boundary_store_stats.stored += 3 - sched.reset() - - stats = sched.get_stats() - assert _values(stats, SAVE_OUTCOMES) == {("stored",): 3} - - -def test_manager_get_stats_gauges_and_info_labels() -> None: - fixture = make_scheduler(num_cpu_blocks=8) - sched = fixture.scheduler - - sched._store_event_to_blocks[0] = TransferMeta([10], [11]) - sched._pending_finished_stores.append(TransferMeta([12], [13])) - sched._abandoned_store_event_to_blocks[1] = TransferMeta([14], [15]) - - stats = sched.get_stats() - assert _values(stats, PENDING_STORE_BLOCKS) == {(): 3} - used = _values(stats, USED_BLOCKS)[()] - assert used == sched.num_cpu_blocks - sched.cpu_block_pool.get_num_free_blocks() - - info = _values(stats, INFO) - assert info == {("cpu", "false", "false", str(sched.num_cpu_blocks)): 1} - - # Gauges are re-reported every drain. - stats2 = sched.get_stats() - assert _values(stats2, PENDING_STORE_BLOCKS) == {(): 3} - - -def test_manager_info_labels_reflect_disk_mode() -> None: - connector = _make_connector( - extra_config={ - "kv_offload_backend": "disk", - "disk_path": "/tmp/fake-disk-metrics", - "disk_capacity_bytes": 1024**3, - "use_page_cache": True, - } - ) - sched = connector.scheduler_manager - assert sched is not None - assert sched._info_labelvalues[0] == "disk" - assert sched._info_labelvalues[1] == "true" - stats = sched.get_stats() - assert set(_values(stats, INFO)) == {sched._info_labelvalues} - - -def test_manager_info_labels_page_cache_false_for_cpu_backend() -> None: - """CPU backend ignores use_page_cache; the info gauge must not claim it.""" - connector = _make_connector(extra_config={"use_page_cache": True}) - sched = connector.scheduler_manager - assert sched is not None - assert sched._info_labelvalues[:2] == ("cpu", "false") - stats = sched.get_stats() - assert set(_values(stats, INFO)) == {sched._info_labelvalues} - - -def test_manager_counts_load_blocks_completed() -> None: - fixture = make_scheduler(num_cpu_blocks=8) - sched = fixture.scheduler - - request = make_request(num_blocks=2) - gpu_blocks = fixture.gpu_block_pool.get_new_blocks(2) - cpu_blocks = sched.cpu_block_pool.get_new_blocks(2) - sched._reqs_to_load[request.request_id] = LoadRequestState( - request=request, - transfer_meta=TransferMeta( - [b.block_id for b in gpu_blocks], [b.block_id for b in cpu_blocks] - ), - ) - - sched.build_connector_meta(make_scheduler_output({})) - stats = sched.get_stats() - assert LOAD_BLOCKS not in stats.data[_StatsKey.DATA] - - sched.update_connector_output( - KVConnectorOutput(finished_recving={request.request_id}) - ) - stats = sched.get_stats() - assert _values(stats, LOAD_BLOCKS) == {(): 2} - - -def test_manager_abandoned_load_counts_as_completed_after_reset() -> None: - """Loads abandoned by reset() that still finish on the worker are - counted as loaded blocks. - """ - fixture = make_scheduler(num_cpu_blocks=8) - sched = fixture.scheduler - - request = make_request(num_blocks=2) - gpu_blocks = fixture.gpu_block_pool.get_new_blocks(2) - cpu_blocks = sched.cpu_block_pool.get_new_blocks(2) - sched._reqs_to_load[request.request_id] = LoadRequestState( - request=request, - transfer_meta=TransferMeta( - [b.block_id for b in gpu_blocks], [b.block_id for b in cpu_blocks] - ), - ) - - sched.build_connector_meta(make_scheduler_output({})) - stats = sched.get_stats() - assert LOAD_BLOCKS not in stats.data[_StatsKey.DATA] - - assert sched.reset() is False - assert set(sched._abandoned_reqs_to_load) == {request.request_id} - - sched.update_connector_output( - KVConnectorOutput(finished_recving={request.request_id}) - ) - assert sched._abandoned_reqs_to_load == {} - stats = sched.get_stats() - assert _values(stats, LOAD_BLOCKS) == {(): 2} diff --git a/tests/v1/kv_connector/unit/test_tp_mapping.py b/tests/v1/kv_connector/unit/test_tp_mapping.py deleted file mode 100644 index e3c73fb2c8..0000000000 --- a/tests/v1/kv_connector/unit/test_tp_mapping.py +++ /dev/null @@ -1,316 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Tests for TP mapping and transfer plan utilities. - -These tests verify that TP mapping produces correct outputs -(source ranks, split handles, desc IDs). -No GPU or NIXL required. -""" - -from __future__ import annotations - -from types import SimpleNamespace - -import numpy as np -import pytest - -from vllm.distributed.kv_transfer.kv_connector.utils import TransferTopology -from vllm.distributed.kv_transfer.kv_connector.v1.nixl.tp_mapping import ( - TPMapping, - compute_tp_mapping, -) -from vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker import ( - NixlConnectorWorker, -) -from vllm.v1.kv_cache_interface import FullAttentionSpec, MambaSpec - -# ====================================================================== -# Test fixtures / helpers -# ====================================================================== - - -def _compute_mapping( - tp_rank: int = 0, - tp_size: int = 1, - remote_tp_size: int = 1, - is_mla: bool = False, - num_kv_heads: int = 8, - group_spec_types: tuple[type, ...] = (FullAttentionSpec,), - dcp_size: int = 1, - remote_dcp_size: int = 1, -) -> TPMapping: - transfer_topology = object.__new__(TransferTopology) - transfer_topology.tp_rank = tp_rank - transfer_topology.tp_size = tp_size - transfer_topology.is_mla = is_mla - transfer_topology.total_num_kv_heads = num_kv_heads - transfer_topology.dcp_size = dcp_size - return compute_tp_mapping( - transfer_topology=transfer_topology, - remote_tp_size=remote_tp_size, - group_spec_types=group_spec_types, - remote_dcp_size=remote_dcp_size, - ) - - -# ====================================================================== -# TP mapping structure tests -# ====================================================================== - - -class TestTPMappingStructure: - def test_source_ranks_homogeneous(self): - m = _compute_mapping(tp_size=2, tp_rank=1, remote_tp_size=2) - assert m.all_source_ranks == (1,) - - def test_source_ranks_d_gt_p(self): - m = _compute_mapping(tp_size=4, tp_rank=2, remote_tp_size=2) - assert m.all_source_ranks == (1,) - - def test_source_ranks_p_gt_d(self): - m = _compute_mapping(tp_size=1, tp_rank=0, remote_tp_size=2) - assert m.all_source_ranks == (0, 1) - - -@pytest.mark.parametrize( - "tp_rank,tp_size,remote_tp_size,dcp_size,remote_dcp_size,expected_ranks", - [ - (0, 1, 8, 1, 8, tuple(range(8))), - (0, 4, 4, 1, 4, (0, 1, 2, 3)), - (2, 4, 4, 4, 4, (2,)), - (0, 2, 4, 2, 4, (0, 2)), - (3, 4, 2, 4, 2, (1,)), - ], -) -def test_mla_dcp_source_ranks( - tp_rank, - tp_size, - remote_tp_size, - dcp_size, - remote_dcp_size, - expected_ranks, -): - mapping = _compute_mapping( - tp_rank=tp_rank, - tp_size=tp_size, - remote_tp_size=remote_tp_size, - is_mla=True, - num_kv_heads=1, - dcp_size=dcp_size, - remote_dcp_size=remote_dcp_size, - ) - - assert mapping.all_source_ranks == expected_ranks - - -@pytest.mark.parametrize( - "tp_size,remote_tp_size,dcp_size,remote_dcp_size", - [ - (4, 2, 4, 2), - (4, 4, 4, 4), - (4, 1, 4, 1), - (4, 4, 1, 4), - (2, 4, 2, 4), - ], -) -def test_dcp_consumer_count_matches_readers( - tp_size, remote_tp_size, dcp_size, remote_dcp_size -): - """Each producer waits for exactly the local ranks that read from it.""" - mappings = [ - _compute_mapping( - tp_rank=tp_rank, - tp_size=tp_size, - remote_tp_size=remote_tp_size, - is_mla=True, - num_kv_heads=1, - dcp_size=dcp_size, - remote_dcp_size=remote_dcp_size, - ) - for tp_rank in range(tp_size) - ] - - for remote_rank in range(remote_tp_size): - readers = [m for m in mappings if remote_rank in m.all_source_ranks] - for mapping in readers: - assert mapping.local_consumers == len(readers) - - -# ====================================================================== -# Split handle tests -# ====================================================================== - - -def _make_mock_worker_for_splits(group_spec_types): - """Build a mock NixlConnectorWorker with _group_spec_types for split tests. - - No per-region replicate flags are configured (``block_len_per_layer`` empty - and ``num_regions == 0``), so ``_build_local_splits_from_plan`` treats every - FA descriptor as SPLIT, matching the legacy behavior these tests assert. - """ - worker = object.__new__(NixlConnectorWorker) - worker._group_spec_types = group_spec_types - worker.transfer_topo = SimpleNamespace(virtually_split_kv_in_blocks=False) - worker.block_len_per_layer = [] - worker.num_regions = 0 - worker._region_is_mla = [] - worker._conv_decomp = SimpleNamespace(local_conv_offsets=()) - worker._ssm_region_indices = [0] if MambaSpec in group_spec_types else [] - worker._ple_region_index = None - return worker - - -class TestBuildSrcSplitHandles: - @pytest.mark.parametrize("remote_tp_size", [2, 4]) - def test_build_src_split_handles(self, remote_tp_size): - tp_rank = 0 - tp_size = 1 - - plan = _compute_mapping( - tp_rank=tp_rank, - tp_size=tp_size, - remote_tp_size=remote_tp_size, - ) - - worker = _make_mock_worker_for_splits((FullAttentionSpec,)) - # One region of 8 dense 1024-byte blocks as a single strided run. - src_blocks_data = np.array([(0x2000, 1024, 0, 1024, 8)], dtype=np.uint64) - num_descs = 8 - splits = list( - worker._build_local_splits_from_plan( - plan, - src_blocks_data, - num_descs, - ) - ) - - assert len(splits) == remote_tp_size - for handle in splits: - assert handle.shape == src_blocks_data.shape - # The head slice shrinks the length; block stride and count are kept. - assert handle[:, 1].tolist() == [1024 // remote_tp_size] - assert handle[:, 3:].tolist() == [[1024, 8]] - - -class TestMambaPlanSplitHandles: - """Verify split handles for Mamba with FA/SSM distinction.""" - - def test_fa_and_ssm_different_split_factors(self): - """Section 0 split by num_attn_reads, section 1 by abs_tp.""" - fa_readers = (0,) - ssm_readers = (0, 1) - plan = TPMapping( - source_ranks_per_group=(fa_readers, ssm_readers), - all_source_ranks=(0, 1), - rank_to_attention_slot={0: 0, 1: 0}, - rank_offset_factor=0, - ) - - worker = _make_mock_worker_for_splits((FullAttentionSpec, MambaSpec)) - # 2 FA runs + 1 SSM run of one block each: (addr, len, dev, stride, count) - src_blocks_data = np.array( - [ - (1000, 200, 0, 200, 1), # FA desc 0 - (2000, 200, 0, 200, 1), # FA desc 1 - (3000, 400, 0, 400, 1), # SSM desc 0 - ], - dtype=np.uint64, - ) - - splits = list(worker._build_local_splits_from_plan(plan, src_blocks_data, 2)) - - assert len(splits) == 2 # 2 source ranks - - # Rank 0 (FA source, p_idx=0): - # FA: chunk=200//1=200, slot=0 → (1000, 200), (2000, 200) - # SSM: chunk=400//2=200, idx=0 → (3000, 200); block stride stays 400 - assert splits[0].tolist() == [ - [1000, 200, 0, 200, 1], - [2000, 200, 0, 200, 1], - [3000, 200, 0, 400, 1], - ] - - # Rank 1 (not FA source, p_idx=1): - # FA: chunk=200//1=200, slot=0 (skip_fa) → (1000, 200), (2000, 200) - # SSM: chunk=400//2=200, idx=1 → (3200, 200); block stride stays 400 - assert splits[1].tolist() == [ - [1000, 200, 0, 200, 1], - [2000, 200, 0, 200, 1], - [3200, 200, 0, 400, 1], - ] - - def test_hetero_block_size_splits(self): - """With a block-size ratio, single-source FA sub-block descs pass - through whole; SSM descs are unexpanded and split per source.""" - plan = TPMapping( - source_ranks_per_group=((0,), (0, 1)), - all_source_ranks=(0, 1), - rank_to_attention_slot={0: 0, 1: 0}, - rank_offset_factor=0, - ) - - worker = _make_mock_worker_for_splits((FullAttentionSpec, MambaSpec)) - # 2 FA blocks x ratio 2 sub-blocks (one run of 2 per block) + 1 SSM desc - # (never expanded). - src_blocks_data = np.array( - [ - (1000, 100, 0, 100, 2), - (2000, 100, 0, 100, 2), - (3000, 400, 0, 400, 1), - ], - dtype=np.uint64, - ) - - splits = list(worker._build_local_splits_from_plan(plan, src_blocks_data, 4, 2)) - - assert len(splits) == 2 - fa_passthrough = [[1000, 100, 0, 100, 2], [2000, 100, 0, 100, 2]] - assert splits[0].tolist() == fa_passthrough + [[3000, 200, 0, 400, 1]] - assert splits[1].tolist() == fa_passthrough + [[3200, 200, 0, 400, 1]] - - def test_hetero_block_size_head_sharded_asserts(self): - """Head-sharded FA reads (multiple FA sources) are incompatible with - a block-size mismatch and must fail loudly.""" - plan = TPMapping( - source_ranks_per_group=((0, 1), (0, 1)), - all_source_ranks=(0, 1), - rank_to_attention_slot={0: 0, 1: 1}, - rank_offset_factor=0, - ) - - worker = _make_mock_worker_for_splits((FullAttentionSpec, MambaSpec)) - src_blocks_data = np.array( - [(1000, 100, 0, 100, 2), (3000, 400, 0, 400, 1)], - dtype=np.uint64, - ) - - with pytest.raises(AssertionError, match="Head-sharded"): - list(worker._build_local_splits_from_plan(plan, src_blocks_data, 2, 2)) - - -@pytest.mark.parametrize( - ("total_kv_heads", "local_tp", "remote_tp", "compatible"), - [ - # 2 heads: TP1 packs both heads into a page, TP >= 2 packs one. - (2, 1, 2, False), - (2, 2, 1, False), - (2, 1, 1, True), - (2, 2, 4, True), - (2, 4, 2, True), - # 4 heads: the boundary moves with the head count. - (4, 2, 4, False), - (4, 2, 2, True), - (4, 4, 8, True), - ], -) -def test_csa_linear_tp_layout_boundary(total_kv_heads, local_tp, remote_tp, compatible): - worker = object.__new__(NixlConnectorWorker) - worker.world_size = local_tp - worker._is_csa_linear = True - worker.transfer_topo = SimpleNamespace(total_num_kv_heads=total_kv_heads) - - if compatible: - worker._validate_csa_linear_tp_layout(remote_tp) - else: - with pytest.raises(ValueError, match="KV-head sharding boundary"): - worker._validate_csa_linear_tp_layout(remote_tp) diff --git a/tests/v1/kv_connector/unit/utils.py b/tests/v1/kv_connector/unit/utils.py index 1e7bfeb14f..df40c97970 100644 --- a/tests/v1/kv_connector/unit/utils.py +++ b/tests/v1/kv_connector/unit/utils.py @@ -537,154 +537,6 @@ def make_kv_cache_config( ) -def make_nixl_scheduler( - has_mamba: bool = False, - is_hma_required: bool = False, - bounded_replay: bool = False, - heartbeat: bool = False, - kv_lease_duration: int = 30, -): - """Create a NixlConnectorScheduler via __new__ (skipping __init__). - - Only sets the flags needed by the tests. When *heartbeat=True* the - scheduler-side heartbeat bookkeeping fields are also initialised. - """ - from types import SimpleNamespace - - from vllm.distributed.kv_transfer.kv_connector.v1.nixl.scheduler import ( - NixlConnectorScheduler, - ) - - sched = object.__new__(NixlConnectorScheduler) - sched._has_mamba = has_mamba - sched._bounded_replay = bounded_replay - sched._is_hma_required = is_hma_required - sched.kv_cache_config = make_kv_cache_config( - block_size=16, - mamba_enabled=has_mamba, - ) - sched.vllm_config = SimpleNamespace(num_prefill_lookahead_tokens=0) - - if heartbeat: - sched._heartbeat_by_engine = {} - sched._heartbeat_req_engine = {} - sched._last_heartbeat_time = 0.0 - sched._kv_lease_duration = kv_lease_duration - sched._heartbeat_interval = kv_lease_duration // 6 - # Fields touched by build_connector_meta / request_finished: - sched._reqs_need_recv = {} - sched._hisparse_host_blocks_to_recv = {} - sched._reqs_need_send = {} - sched._reqs_in_batch = set() - sched._reqs_not_processed = set() - sched._reqs_need_save = {} - sched.use_host_buffer = False - sched.engine_id = "test-engine" - sched.transfer_tp_size = 1 - sched.side_channel_host = "localhost" - sched.side_channel_port = 5555 - sched.blocks_per_sw = [] - sched.is_bidirectional_kv_xfer_enabled = False - return sched - - -def make_nixl_push_scheduler( - *, - decoder_kv_blocks_ttl: float = 30.0, - push_registration_timeout: float | None = None, - is_bidirectional_kv_xfer_enabled: bool = False, - has_mamba: bool = False, -): - """Create a NixlPushConnectorScheduler via __new__ (skipping __init__). - - The push scheduler can't reuse :func:`make_nixl_scheduler` because it - is a different class (``NixlPushConnectorScheduler`` vs - ``NixlConnectorScheduler``) and carries push-specific state. Only the - fields touched by the unit tests are populated. - """ - from unittest.mock import MagicMock - - from vllm.distributed.kv_transfer.kv_connector.v1.nixl.push_scheduler import ( - NixlPushConnectorScheduler, - ) - - sched = object.__new__(NixlPushConnectorScheduler) - - # Base scheduler fields (shared with pull / heartbeat path). - sched._reqs_need_recv = {} - sched._reqs_need_send = {} - sched._reqs_in_batch = set() - sched._reqs_not_processed = set() - sched._reqs_need_save = {} - sched._kv_lease_duration = 30 - sched.decoder_kv_blocks_ttl = decoder_kv_blocks_ttl - sched.use_host_buffer = False - sched.engine_id = "decode-engine" - sched.transfer_tp_size = 1 - sched.side_channel_host = "127.0.0.1" - sched.side_channel_port = 5600 - sched.is_bidirectional_kv_xfer_enabled = is_bidirectional_kv_xfer_enabled - sched._has_mamba = has_mamba - sched._bounded_replay = False - sched.kv_cache_config = make_kv_cache_config( - block_size=16, - mamba_enabled=has_mamba, - ) - - # vllm_config is consulted for parallel_config.tensor_parallel_size, and by - # `_prefill_backoff` on both the P and D prefill paths. - vllm_config = MagicMock() - vllm_config.parallel_config.tensor_parallel_size = 1 - vllm_config.num_prefill_lookahead_tokens = 0 - sched.vllm_config = vllm_config - - # Push-specific state. - sched._push_pending_registrations = {} - sched._push_registration_deadlines = {} - sched._finished_request_blocks = {} - sched._newly_finished_push_blocks = {} - sched._push_registration_timeout = ( - push_registration_timeout - if push_registration_timeout is not None - else decoder_kv_blocks_ttl - ) - - # Heartbeat fields touched by base request_finished / - # update_connector_output. - sched._heartbeat_by_engine = {} - sched._heartbeat_req_engine = {} - sched._last_heartbeat_time = 0.0 - sched.blocks_per_sw = [] - - return sched - - -def make_moriio_writer(fake_worker: Any) -> Any: - """Build a MoRIIOWriter with internals stubbed for unit tests. - - Bypasses ``__init__`` and wires only the write/finalize state the tests - touch, including the deferred-task fields used by the routing suite. - """ - import threading - from queue import Queue - - from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_engine import ( - MoRIIOWriter, - ) - - writer = MoRIIOWriter.__new__(MoRIIOWriter) - writer._worker_ref = lambda: fake_worker - writer._write_task_q = Queue() - writer._write_state_lock = threading.Lock() - writer._scheduled_writes = defaultdict(int) - writer._scheduled_layers = defaultdict(set) - writer._sealed_writes = {} - writer._deferred_tasks = [] - writer._defer_timeout = 60.0 - writer.ensure_worker_started = lambda: None - return writer - - def expand_strided_descs(strided_descs: np.ndarray) -> np.ndarray: """Expand Nx5 (addr, len, dev, stride, count) rows to one row per block. diff --git a/tests/v1/simple_kv_offload/test_scheduler.py b/tests/v1/simple_kv_offload/test_scheduler.py deleted file mode 100644 index 31aa75d31f..0000000000 --- a/tests/v1/simple_kv_offload/test_scheduler.py +++ /dev/null @@ -1,3395 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Unit tests for SimpleCPUOffloadScheduler.""" - -from __future__ import annotations - -import logging -from dataclasses import dataclass -from unittest.mock import Mock - -import pytest -import torch - -from vllm import SamplingParams -from vllm.config import ( - CacheConfig, - DeviceConfig, - KVTransferConfig, - ModelConfig, - SchedulerConfig, - SpeculativeConfig, - VllmConfig, -) -from vllm.config.cache import MambaCacheMode -from vllm.config.kv_events import KVEventsConfig -from vllm.distributed.kv_transfer.kv_connector.v1 import KVConnectorRole -from vllm.distributed.kv_transfer.kv_connector.v1.simple_cpu_offload_connector import ( - SimpleCPUOffloadConnector, -) -from vllm.utils.hashing import sha256 -from vllm.v1.core.block_pool import BlockPool -from vllm.v1.core.kv_cache_manager import KVCacheBlocks, KVCacheManager -from vllm.v1.core.kv_cache_utils import ( - get_request_block_hasher, - init_none_hash, - make_block_hash_with_group_id, -) -from vllm.v1.core.sched.output import ( - CachedRequestData, - KVConnectorBlockState, - NewRequestData, - SchedulerOutput, -) -from vllm.v1.core.single_type_kv_cache_manager import ( - register_all_kvcache_specs, -) -from vllm.v1.kv_cache_interface import ( - CircularBufferSpec, - FullAttentionSpec, - KVCacheConfig, - KVCacheGroupSpec, - KVCacheSpec, - KVCacheTensor, - MambaSpec, - SlidingWindowSpec, -) -from vllm.v1.metrics.cache_hit_source import CacheHitSource -from vllm.v1.outputs import KVConnectorOutput -from vllm.v1.request import Request -from vllm.v1.simple_kv_offload.manager import SimpleCPUOffloadScheduler -from vllm.v1.simple_kv_offload.metadata import SimpleCPUOffloadWorkerMetadata - -pytestmark = pytest.mark.skip_global_cleanup - - -# --------------------------------------------------------------------------- -# Constants -# --------------------------------------------------------------------------- -BLOCK_SIZE = 16 -HEAD_SIZE = 16 -NUM_KV_HEADS = 1 -DTYPE = torch.float16 -# bytes per block per tensor: -# block_size * num_kv_heads * head_size * 2 (K+V) * element_size -_BYTES_PER_BLOCK = BLOCK_SIZE * NUM_KV_HEADS * HEAD_SIZE * 2 * DTYPE.itemsize - -# Ensure none_hash is initialized once -init_none_hash(sha256) - - -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- - - -def _make_kv_cache_config( - num_blocks: int, - num_groups: int = 1, -) -> KVCacheConfig: - """Build a KVCacheConfig with non-empty kv_cache_tensors.""" - groups = [] - tensors = [] - register_all_kvcache_specs( - vllm_config=None - ) # Ensure specs are registered for tests - for g in range(num_groups): - layer_names = [f"layer_{g}"] - spec = ( - FullAttentionSpec( - block_size=BLOCK_SIZE, - num_kv_heads=NUM_KV_HEADS, - head_size=HEAD_SIZE, - dtype=DTYPE, - ) - if g == 0 - else SlidingWindowSpec( - block_size=BLOCK_SIZE, - num_kv_heads=NUM_KV_HEADS, - head_size=HEAD_SIZE, - dtype=DTYPE, - sliding_window=BLOCK_SIZE * 4, - ) - ) - groups.append( - KVCacheGroupSpec( - layer_names, - spec, - ) - ) - tensors.append( - KVCacheTensor( - size=_BYTES_PER_BLOCK * num_blocks, - layers=layer_names, - layer_stride=_BYTES_PER_BLOCK * num_blocks, - block_stride=_BYTES_PER_BLOCK, - ) - ) - return KVCacheConfig( - num_blocks=num_blocks, - kv_cache_tensors=tensors, - kv_cache_groups=groups, - ) - - -def _make_scratch_kv_cache_config( - num_blocks: int, scratch_block_size: int = 4 -) -> KVCacheConfig: - """FullAttention group plus a non-prefix-cacheable scratch group. - - Mirrors GLM-5.3-Flash, whose kpool-tail group holds one per-request - scratch block of ``index_kpool`` tokens: it is excluded from prefix - caching, so its block size neither divides nor is divided by the - hash block size in general. - """ - register_all_kvcache_specs(vllm_config=None) - fa_config = _make_kv_cache_config(num_blocks, num_groups=1) - scratch_layers = ["layer_scratch"] - scratch_spec = CircularBufferSpec( - block_size=scratch_block_size, - num_kv_heads=2, - head_size=HEAD_SIZE, - head_size_v=0, - dtype=DTYPE, - ) - assert not scratch_spec.prefix_cacheable - scratch_bytes = scratch_spec.page_size_bytes * num_blocks - return KVCacheConfig( - num_blocks=num_blocks, - kv_cache_tensors=fa_config.kv_cache_tensors - + [ - KVCacheTensor( - size=scratch_bytes, - layers=scratch_layers, - layer_stride=scratch_bytes, - block_stride=scratch_spec.page_size_bytes, - ) - ], - kv_cache_groups=fa_config.kv_cache_groups - + [KVCacheGroupSpec(scratch_layers, scratch_spec)], - ) - - -def _make_vllm_config(block_size: int = BLOCK_SIZE) -> VllmConfig: - """Minimal VllmConfig for scheduler tests (no GPU).""" - model_config = ModelConfig( - model="facebook/opt-125m", - trust_remote_code=True, - dtype="float16", - seed=42, - ) - scheduler_config = SchedulerConfig( - max_num_seqs=16, - max_num_batched_tokens=64, - max_model_len=10000, - enable_chunked_prefill=True, - is_encoder_decoder=False, - ) - cache_config = CacheConfig( - block_size=block_size, - gpu_memory_utilization=0.9, - enable_prefix_caching=True, - ) - kv_transfer_config = KVTransferConfig( - kv_connector="SimpleCPUOffloadConnector", - kv_role="kv_both", - ) - return VllmConfig( - scheduler_config=scheduler_config, - model_config=model_config, - cache_config=cache_config, - kv_transfer_config=kv_transfer_config, - device_config=DeviceConfig("cpu"), - ) - - -@dataclass -class SchedulerFixture: - """Bundle returned by make_scheduler for convenient access.""" - - scheduler: SimpleCPUOffloadScheduler - gpu_block_pool: BlockPool - vllm_config: VllmConfig - kv_cache_config: KVCacheConfig - num_groups: int = 1 - - -@pytest.mark.parametrize( - ("backend", "source"), - [("cpu", CacheHitSource.HOST), ("disk", CacheHitSource.DISK)], -) -def test_connector_reports_configured_cache_source( - tmp_path, backend: str, source: CacheHitSource -): - kv_cache_config = _make_kv_cache_config(num_blocks=4) - vllm_config = _make_vllm_config() - extra_config = { - "cpu_bytes_to_use_per_rank": _BYTES_PER_BLOCK * 4, - "kv_offload_backend": backend, - } - if backend == "disk": - extra_config.update( - disk_path=str(tmp_path / "kv-offload.bin"), - disk_capacity_bytes=_BYTES_PER_BLOCK * 4, - ) - assert vllm_config.kv_transfer_config is not None - vllm_config.kv_transfer_config.kv_connector_extra_config = extra_config - - connector = SimpleCPUOffloadConnector( - vllm_config, - KVConnectorRole.SCHEDULER, - kv_cache_config, - ) - - assert connector.get_external_cache_hit_sources(None, 32) == {source: 32} # type: ignore[arg-type] - assert connector.get_external_cache_hit_sources(None, 0) == {} # type: ignore[arg-type] - - -@pytest.mark.parametrize("num_layers", [1, 2]) -@pytest.mark.parametrize("backend", ["cpu", "disk"]) -def test_connector_explicit_cpu_blocks_align_uneven_stages( - tmp_path, num_layers: int, backend: str -): - """The same configured pool size must fit lighter and heavier PP stages.""" - config = _make_kv_cache_config(num_blocks=16) - names = [f"layer_{i}" for i in range(num_layers)] - config.kv_cache_groups[0].layer_names = names - config.kv_cache_tensors[0].layers = names - config.kv_cache_tensors[0].size *= num_layers - vllm_config = _make_vllm_config() - extra = { - "cpu_bytes_to_use_per_rank": _BYTES_PER_BLOCK * 8, - "num_cpu_blocks": 4, - "kv_offload_backend": backend, - } - if backend == "disk": - extra.update( - disk_path=str(tmp_path / "offload.bin"), - disk_capacity_bytes=_BYTES_PER_BLOCK * 8, - ) - assert vllm_config.kv_transfer_config is not None - vllm_config.kv_transfer_config.kv_connector_extra_config = extra - connector = SimpleCPUOffloadConnector( - vllm_config, KVConnectorRole.SCHEDULER, config - ) - assert connector.scheduler_manager is not None - assert connector.scheduler_manager.num_cpu_blocks == 4 - assert len(connector.scheduler_manager.cpu_block_pool.blocks) == 4 - - -@pytest.mark.parametrize( - "count, capacity_bytes", - [(count, _BYTES_PER_BLOCK * 8) for count in [0, -1, True, 1.5, "4", 9]] - + [(1, _BYTES_PER_BLOCK // 2)], -) -def test_connector_rejects_invalid_explicit_cpu_blocks(count, capacity_bytes): - config = _make_kv_cache_config(num_blocks=16) - vllm_config = _make_vllm_config() - assert vllm_config.kv_transfer_config is not None - vllm_config.kv_transfer_config.kv_connector_extra_config = { - "cpu_bytes_to_use_per_rank": capacity_bytes, - "num_cpu_blocks": count, - } - with pytest.raises(ValueError, match="num_cpu_blocks"): - SimpleCPUOffloadConnector(vllm_config, KVConnectorRole.SCHEDULER, config) - - -def make_scheduler( - num_cpu_blocks: int = 8, - num_gpu_blocks: int = 16, - num_groups: int = 1, - lazy: bool = False, - kv_cache_config: KVCacheConfig | None = None, -) -> SchedulerFixture: - """Build a SimpleCPUOffloadScheduler with small block pools.""" - if kv_cache_config is None: - kv_cache_config = _make_kv_cache_config(num_gpu_blocks, num_groups) - else: - num_groups = len(kv_cache_config.kv_cache_groups) - vllm_config = _make_vllm_config() - cpu_capacity_bytes = _BYTES_PER_BLOCK * num_cpu_blocks * num_groups - - sched = SimpleCPUOffloadScheduler( - vllm_config=vllm_config, - kv_cache_config=kv_cache_config, - cpu_capacity_bytes=cpu_capacity_bytes, - scheduler_block_size=BLOCK_SIZE, - hash_block_size=BLOCK_SIZE, - lazy_offload=lazy, - ) - - # Build a real GPU block pool and bind it - gpu_block_pool = BlockPool( - num_gpu_blocks=num_gpu_blocks, - enable_caching=True, - hash_block_size=BLOCK_SIZE, - ) - sched.bind_gpu_block_pool(gpu_block_pool) - - return SchedulerFixture( - scheduler=sched, - gpu_block_pool=gpu_block_pool, - vllm_config=vllm_config, - kv_cache_config=kv_cache_config, - num_groups=num_groups, - ) - - -_req_counter = 0 - - -def make_request( - num_blocks: int = 2, - request_id: str | None = None, - extra_tokens: int = 1, -) -> Request: - """Create a Request with deterministic block hashes.""" - global _req_counter - _req_counter += 1 - if request_id is None: - request_id = f"req-{_req_counter}" - - num_tokens = num_blocks * BLOCK_SIZE + extra_tokens - start = _req_counter * 10000 - prompt_token_ids = list(range(start, start + num_tokens)) - sampling_params = SamplingParams(max_tokens=1) - - req = Request( - request_id=request_id, - prompt_token_ids=prompt_token_ids, - sampling_params=sampling_params, - pooling_params=None, - mm_features=None, - block_hasher=get_request_block_hasher(BLOCK_SIZE, sha256), - ) - return req - - -def make_scheduler_output( - req_id_to_num_tokens: dict[str, int], - *, - new_reqs: dict[str, tuple[list[int], ...]] | None = None, - cached_req_new_blocks: dict[str, tuple[list[int], ...] | None] | None = None, -) -> SchedulerOutput: - """Build a minimal SchedulerOutput with num_scheduled_tokens. - - Args: - new_reqs: For first-time requests, maps req_id -> block_ids tuple. - These are placed into ``scheduled_new_reqs`` as ``NewRequestData``. - cached_req_new_blocks: For returning (cached) requests, maps - req_id -> new_block_ids (incremental) or None. - These are placed into ``scheduled_cached_reqs``. - - """ - scheduled_new_reqs: list[NewRequestData] = [] - if new_reqs: - for req_id, block_ids in new_reqs.items(): - scheduled_new_reqs.append( - NewRequestData( - req_id=req_id, - prompt_token_ids=None, - mm_features=[], - sampling_params=None, - pooling_params=None, - block_ids=block_ids, - num_computed_tokens=0, - lora_request=None, - ) - ) - - if cached_req_new_blocks: - cached_req_ids = list(cached_req_new_blocks.keys()) - cached_new_block_ids = [cached_req_new_blocks[rid] for rid in cached_req_ids] - cached_reqs = CachedRequestData( - req_ids=cached_req_ids, - resumed_req_ids=set(), - new_token_ids=[[] for _ in cached_req_ids], - all_token_ids={}, - new_block_ids=cached_new_block_ids, - num_computed_tokens=[0] * len(cached_req_ids), - num_output_tokens=[0] * len(cached_req_ids), - ) - else: - cached_reqs = CachedRequestData.make_empty() - - return SchedulerOutput( - scheduled_new_reqs=scheduled_new_reqs, - scheduled_cached_reqs=cached_reqs, - num_scheduled_tokens=req_id_to_num_tokens, - total_num_scheduled_tokens=sum(req_id_to_num_tokens.values()), - scheduled_spec_decode_tokens={}, - scheduled_encoder_inputs={}, - preempted_req_ids=set(), - finished_req_ids=set(), - free_encoder_mm_hashes=[], - ) - - -def simulate_store_completion( - scheduler: SimpleCPUOffloadScheduler, - event_idx: int, -) -> None: - """Simulate worker reporting a store event completion.""" - output = KVConnectorOutput( - finished_recving=set(), - kv_connector_worker_meta=SimpleCPUOffloadWorkerMetadata( - completed_store_events={event_idx: scheduler._expected_worker_count}, - ), - ) - scheduler.update_connector_output(output) - - -def simulate_load_completion( - scheduler: SimpleCPUOffloadScheduler, - req_ids: set[str], -) -> None: - """Simulate worker reporting load completions for requests.""" - output = KVConnectorOutput( - finished_sending=set(), - finished_recving=req_ids, - ) - scheduler.update_connector_output(output) - - -def get_cpu_free_blocks(scheduler: SimpleCPUOffloadScheduler) -> int: - """Return number of free CPU blocks.""" - return scheduler.cpu_block_pool.get_num_free_blocks() - - -def _allocate_gpu_blocks( - gpu_block_pool: BlockPool, - request: Request, - num_blocks: int, - group_id: int = 0, -) -> list: - """Allocate GPU blocks, cache them with hashes, return block list. - - Mimics what KVCacheManager does: allocate blocks from pool, then - register them in the prefix cache via cache_full_blocks so that - re-allocation properly evicts stale hashes. - """ - blocks = gpu_block_pool.get_new_blocks(num_blocks) - num_full = min(num_blocks, len(request.block_hashes)) - if num_full > 0: - gpu_block_pool.cache_full_blocks( - request=request, - blocks=blocks, - num_cached_blocks=0, - num_full_blocks=num_full, - block_size=BLOCK_SIZE, - kv_cache_group_id=group_id, - ) - return blocks - - -def _alloc_and_register( - fix: SchedulerFixture, - request: Request, - num_blocks: int, - *, - confirmed: bool = True, - group_id: int = 0, -) -> KVCacheBlocks: - """Allocate GPU blocks and return KVCacheBlocks. - - Block IDs are no longer registered in a mock KVCacheManager; instead - tests pass them through ``make_scheduler_output`` so that - ``yield_req_data`` can pick them up. - - If ``confirmed`` is True, advance ``request.num_computed_tokens`` to simulate - the scheduler's ``_update_after_schedule`` from a prior step. - """ - gpu_blocks = _allocate_gpu_blocks( - fix.gpu_block_pool, request, num_blocks, group_id=group_id - ) - kv_blocks = KVCacheBlocks(blocks=(gpu_blocks,)) - if confirmed: - request.num_computed_tokens = num_blocks * BLOCK_SIZE - return kv_blocks - - -# --------------------------------------------------------------------------- -# Test 1a: Eager store-and-load roundtrip -# --------------------------------------------------------------------------- -def test_eager_store_and_load_roundtrip() -> None: - """Eager mode: store blocks on compute, complete store, verify cache hit.""" - fix = make_scheduler(num_cpu_blocks=8, num_gpu_blocks=16, lazy=False) - sched = fix.scheduler - - num_blocks = 2 - req = make_request(num_blocks=num_blocks) - - kv_blocks = _alloc_and_register(fix, req, num_blocks) - sched.update_state_after_alloc(req, kv_blocks, num_external_tokens=0) - block_ids = kv_blocks.get_block_ids() - sched_out = make_scheduler_output( - {req.request_id: num_blocks * BLOCK_SIZE}, - new_reqs={req.request_id: block_ids}, - ) - - meta = sched.build_connector_meta(sched_out) - assert meta.store_event >= 0, "Expected a store event to be scheduled" - assert len(meta.store_gpu_blocks) > 0 - assert len(meta.store_cpu_blocks) == len(meta.store_gpu_blocks) - simulate_store_completion(sched, meta.store_event) - - # New request with same tokens should get CPU cache hit - req2 = Request( - request_id="req-eager-load", - prompt_token_ids=req.prompt_token_ids, - sampling_params=req.sampling_params, - pooling_params=None, - mm_features=None, - block_hasher=req._block_hasher, - ) - hit_tokens, is_async = sched.get_num_new_matched_tokens(req2, num_computed_tokens=0) - # make_request pads num_tokens by +1 beyond the last full block, so the - # manager's max_hit_len = num_tokens - 1 cap leaves all full blocks intact. - assert hit_tokens == num_blocks * BLOCK_SIZE - assert is_async is True - - gpu_blocks2 = fix.gpu_block_pool.get_new_blocks(num_blocks) - kv_blocks2 = KVCacheBlocks(blocks=(gpu_blocks2,)) - sched.update_state_after_alloc(req2, kv_blocks2, num_external_tokens=hit_tokens) - - block_ids2 = kv_blocks2.get_block_ids() - sched_out2 = make_scheduler_output( - {req2.request_id: 1}, - new_reqs={req2.request_id: block_ids2}, - ) - meta2 = sched.build_connector_meta(sched_out2) - assert meta2.load_event >= 0, "Expected a load event to be assigned" - assert len(meta2.load_gpu_blocks) > 0 - assert len(meta2.load_cpu_blocks) == len(meta2.load_gpu_blocks) - - -def test_prompt_logprobs_skip_cpu_cache_lookup() -> None: - """Prompt logprobs require every prompt token to be recomputed.""" - fix = make_scheduler(num_cpu_blocks=8, num_gpu_blocks=16, lazy=False) - sched = fix.scheduler - - num_blocks = 2 - req = make_request(num_blocks=num_blocks) - kv_blocks = _alloc_and_register(fix, req, num_blocks) - sched.update_state_after_alloc(req, kv_blocks, num_external_tokens=0) - sched_out = make_scheduler_output( - {req.request_id: num_blocks * BLOCK_SIZE}, - new_reqs={req.request_id: kv_blocks.get_block_ids()}, - ) - meta = sched.build_connector_meta(sched_out) - simulate_store_completion(sched, meta.store_event) - - retry_request_id = "req-prompt-logprobs" - cached_request = Request( - request_id=retry_request_id, - prompt_token_ids=req.prompt_token_ids, - sampling_params=req.sampling_params, - pooling_params=None, - mm_features=None, - block_hasher=req._block_hasher, - ) - assert sched.get_num_new_matched_tokens(cached_request, num_computed_tokens=0) == ( - num_blocks * BLOCK_SIZE, - True, - ) - pending_hit = sched._pending_cpu_hits[retry_request_id] - pinned_blocks = [ - block for group in pending_hit[0] for block in group if not block.is_null - ] - assert pinned_blocks - assert all(block.ref_cnt == 1 for block in pinned_blocks) - - prompt_logprobs_request = Request( - request_id=retry_request_id, - prompt_token_ids=req.prompt_token_ids, - sampling_params=SamplingParams(max_tokens=1, prompt_logprobs=1), - pooling_params=None, - mm_features=None, - block_hasher=req._block_hasher, - ) - assert prompt_logprobs_request.skip_reading_prefix_cache - - assert sched.get_num_new_matched_tokens( - prompt_logprobs_request, num_computed_tokens=0 - ) == (0, False) - assert retry_request_id not in sched._pending_cpu_hits - assert all(block.ref_cnt == 0 for block in pinned_blocks) - - -def test_eager_store_preserves_secondary_block_hashes() -> None: - """CPU copies retain fine-grained hashes owned by the GPU block.""" - fix = make_scheduler(num_cpu_blocks=8, num_gpu_blocks=16, lazy=False) - sched = fix.scheduler - req = make_request(num_blocks=1) - kv_blocks = _alloc_and_register(fix, req, num_blocks=1) - gpu_block = kv_blocks.blocks[0][0] - primary_hash = gpu_block.block_hash - primary_num_tokens = gpu_block.block_hash_num_tokens - assert primary_hash is not None - - fine_grained_req = Request( - request_id="req-fine-grained-hash", - prompt_token_ids=req.prompt_token_ids, - sampling_params=req.sampling_params, - pooling_params=None, - mm_features=None, - block_hasher=get_request_block_hasher(BLOCK_SIZE // 2, sha256), - ) - secondary_hash = make_block_hash_with_group_id(fine_grained_req.block_hashes[0], 0) - fix.gpu_block_pool._insert_block_hash( - secondary_hash, gpu_block, num_tokens=BLOCK_SIZE // 2 - ) - assert ( - secondary_hash - in fix.gpu_block_pool.cached_block_hashes_by_block[gpu_block.block_id] - ) - - sched.update_state_after_alloc(req, kv_blocks, num_external_tokens=0) - block_ids = kv_blocks.get_block_ids() - meta = sched.build_connector_meta( - make_scheduler_output( - {req.request_id: BLOCK_SIZE}, - new_reqs={req.request_id: block_ids}, - ) - ) - simulate_store_completion(sched, meta.store_event) - - cpu_block = sched.cpu_block_pool.cached_block_hash_to_block.get_one_block( - secondary_hash - ) - assert cpu_block is not None - assert cpu_block.block_hash == primary_hash - assert cpu_block.block_hash_num_tokens == primary_num_tokens - assert ( - secondary_hash - in sched.cpu_block_pool.cached_block_hashes_by_block[cpu_block.block_id] - ) - - -# --------------------------------------------------------------------------- -# Test 1b: Finished eager stores -# --------------------------------------------------------------------------- -def test_finished_eager_stores_are_batched() -> None: - """Finished requests in one step retain, complete, and cache every store.""" - fix = make_scheduler(num_cpu_blocks=8, num_gpu_blocks=16, lazy=False) - sched = fix.scheduler - - requests = [make_request(num_blocks=2) for _ in range(2)] - block_ids = [] - for request in requests: - kv_blocks = _alloc_and_register(fix, request, num_blocks=2) - sched.update_state_after_alloc(request, kv_blocks, num_external_tokens=0) - block_ids.append(kv_blocks.get_block_ids()) - - for request, request_block_ids in zip(requests, block_ids): - sched.request_finished_all_groups(request, request_block_ids) - - meta = sched.build_connector_meta(make_scheduler_output({})) - assert len(meta.store_gpu_blocks) == 4 - assert len(meta.store_cpu_blocks) == 4 - simulate_store_completion(sched, meta.store_event) - - for request in requests: - for block_hash in request.block_hashes[:2]: - cached = sched.cpu_block_pool.cached_block_hash_to_block.get_one_block( - make_block_hash_with_group_id(block_hash, 0) - ) - assert cached is not None - - request = requests[0] - cached_request = Request( - request_id="req-finished-eager-load", - prompt_token_ids=request.prompt_token_ids, - sampling_params=request.sampling_params, - pooling_params=None, - mm_features=None, - block_hasher=request._block_hasher, - ) - hit_tokens, is_async = sched.get_num_new_matched_tokens( - cached_request, num_computed_tokens=0 - ) - assert hit_tokens == 2 * BLOCK_SIZE - assert is_async is True - - cached_gpu_blocks = fix.gpu_block_pool.get_new_blocks(2) - sched.update_state_after_alloc( - cached_request, - KVCacheBlocks(blocks=(cached_gpu_blocks,)), - num_external_tokens=hit_tokens, - ) - load_meta = sched.build_connector_meta( - make_scheduler_output( - {cached_request.request_id: 1}, - new_reqs={ - cached_request.request_id: ( - [block.block_id for block in cached_gpu_blocks], - ) - }, - ) - ) - assert load_meta.load_event >= 0 - assert len(load_meta.load_gpu_blocks) == 2 - assert len(load_meta.load_cpu_blocks) == 2 - - -def test_finished_eager_store_skips_existing_cpu_cache_entries() -> None: - """A final flush must not consume CPU capacity for an existing hash.""" - fix = make_scheduler(num_cpu_blocks=8, num_gpu_blocks=16, lazy=False) - sched = fix.scheduler - - first = make_request(num_blocks=2) - first_blocks = _alloc_and_register(fix, first, num_blocks=2) - sched.update_state_after_alloc(first, first_blocks, num_external_tokens=0) - sched.request_finished_all_groups(first, first_blocks.get_block_ids()) - first_meta = sched.build_connector_meta(make_scheduler_output({})) - simulate_store_completion(sched, first_meta.store_event) - free_blocks = get_cpu_free_blocks(sched) - - duplicate = Request( - request_id="req-finished-eager-duplicate", - prompt_token_ids=first.prompt_token_ids, - sampling_params=first.sampling_params, - pooling_params=None, - mm_features=None, - block_hasher=first._block_hasher, - ) - duplicate_blocks = _alloc_and_register(fix, duplicate, num_blocks=2) - sched.update_state_after_alloc(duplicate, duplicate_blocks, num_external_tokens=0) - sched.request_finished_all_groups(duplicate, duplicate_blocks.get_block_ids()) - - duplicate_meta = sched.build_connector_meta(make_scheduler_output({})) - assert duplicate_meta.store_event == -1 - assert get_cpu_free_blocks(sched) == free_blocks - - -def test_finished_eager_store_is_reported_pending_before_metadata_build() -> None: - """Queued finished stores must keep the connector pending until submitted.""" - fix = make_scheduler(num_cpu_blocks=8, num_gpu_blocks=16, lazy=False) - sched = fix.scheduler - request = make_request(num_blocks=2) - kv_blocks = _alloc_and_register(fix, request, num_blocks=2) - connector = SimpleCPUOffloadConnector.__new__(SimpleCPUOffloadConnector) - connector.scheduler_manager = sched - sched.update_state_after_alloc(request, kv_blocks, num_external_tokens=0) - - sched.request_finished_all_groups(request, kv_blocks.get_block_ids()) - - assert sched.has_pending_stores() - assert connector.has_pending_push_work() - meta = sched.build_connector_meta(make_scheduler_output({})) - assert meta.store_event >= 0 - assert not sched._pending_finished_stores - assert connector.has_pending_push_work() - simulate_store_completion(sched, meta.store_event) - assert not sched.has_pending_stores() - assert not connector.has_pending_push_work() - - -def test_finished_eager_store_caches_all_groups() -> None: - """The final flush retains completed blocks from every KV cache group.""" - fix = make_scheduler(num_cpu_blocks=8, num_gpu_blocks=16, num_groups=2, lazy=False) - sched = fix.scheduler - request = make_request(num_blocks=2) - group_blocks = tuple( - _allocate_gpu_blocks(fix.gpu_block_pool, request, 2, group_id=group_id) - for group_id in range(2) - ) - request.num_computed_tokens = 2 * BLOCK_SIZE - kv_blocks = KVCacheBlocks(blocks=group_blocks) - sched.update_state_after_alloc(request, kv_blocks, num_external_tokens=0) - - sched.request_finished_all_groups(request, kv_blocks.get_block_ids()) - meta = sched.build_connector_meta(make_scheduler_output({})) - assert len(meta.store_gpu_blocks) == 4 - simulate_store_completion(sched, meta.store_event) - - for group_id in range(2): - for block_hash in request.block_hashes[:2]: - cached = sched.cpu_block_pool.cached_block_hash_to_block.get_one_block( - make_block_hash_with_group_id(block_hash, group_id) - ) - assert cached is not None - - -def test_finished_eager_store_respects_cpu_capacity() -> None: - """A finished request skips excess blocks when the CPU cache is full.""" - fix = make_scheduler(num_cpu_blocks=3, num_gpu_blocks=16, lazy=False) - sched = fix.scheduler - request = make_request(num_blocks=3) - kv_blocks = _alloc_and_register(fix, request, num_blocks=3) - sched.update_state_after_alloc(request, kv_blocks, num_external_tokens=0) - available = get_cpu_free_blocks(sched) - - sched.request_finished_all_groups(request, kv_blocks.get_block_ids()) - meta = sched.build_connector_meta(make_scheduler_output({})) - - assert len(meta.store_gpu_blocks) == available - assert len(meta.store_cpu_blocks) == available - - -def test_reset_releases_unsubmitted_finished_stores() -> None: - """Reset releases finished-store refs that were not sent to a worker.""" - fix = make_scheduler(num_cpu_blocks=8, num_gpu_blocks=16, lazy=False) - sched = fix.scheduler - request = make_request(num_blocks=2) - kv_blocks = _alloc_and_register(fix, request, num_blocks=2) - sched.update_state_after_alloc(request, kv_blocks, num_external_tokens=0) - - sched.request_finished_all_groups(request, kv_blocks.get_block_ids()) - - assert sched.reset() - - -# --------------------------------------------------------------------------- -# Test 1b: Boundary — max_hit_len cap drops the last full block when the -# prompt is an exact multiple of BLOCK_SIZE. -# --------------------------------------------------------------------------- -def test_max_hit_len_cap_drops_last_full_block() -> None: - """When num_tokens is an exact multiple of BLOCK_SIZE, the manager's - ``max_hit_len = num_tokens - 1`` cap forces ``find_longest_cache_hit`` to - drop the final block (since ``max_length // block_size`` rounds down). - """ - fix = make_scheduler(num_cpu_blocks=8, num_gpu_blocks=16, lazy=False) - sched = fix.scheduler - - num_blocks = 2 - req = make_request(num_blocks=num_blocks, extra_tokens=0) - assert req.num_tokens == num_blocks * BLOCK_SIZE - - kv_blocks = _alloc_and_register(fix, req, num_blocks) - sched.update_state_after_alloc(req, kv_blocks, num_external_tokens=0) - sched_out = make_scheduler_output( - {req.request_id: num_blocks * BLOCK_SIZE}, - new_reqs={req.request_id: kv_blocks.get_block_ids()}, - ) - meta = sched.build_connector_meta(sched_out) - simulate_store_completion(sched, meta.store_event) - - req2 = Request( - request_id="req-cap-boundary", - prompt_token_ids=req.prompt_token_ids, - sampling_params=req.sampling_params, - pooling_params=None, - mm_features=None, - block_hasher=req._block_hasher, - ) - hit_tokens, _ = sched.get_num_new_matched_tokens(req2, num_computed_tokens=0) - assert hit_tokens == (num_blocks - 1) * BLOCK_SIZE - - -# --------------------------------------------------------------------------- -# Test 1c: Lazy store-and-load roundtrip -# --------------------------------------------------------------------------- -def _flush_old_blocks_to_lru_head( - gpu_pool: BlockPool, - num_filler_blocks: int, -) -> list: - """Allocate filler blocks so that previously-freed (hashed) blocks migrate - to the LRU head of the free queue. Returns the filler blocks (caller must - free them later to restore pool capacity). - - In a real engine the same thing happens naturally: after one request - finishes and frees its blocks, subsequent requests allocate from the LRU - head, consuming the unhashed blocks and leaving the old hashed blocks at - the front of the queue. - """ - fillers = gpu_pool.get_new_blocks(num_filler_blocks) - return fillers - - -def test_lazy_store_and_load_roundtrip() -> None: - """Lazy mode: schedule a request, finish it so its hashed blocks are freed, - then schedule new requests so the old blocks migrate to the LRU head. - The lazy scanner offloads them to CPU. Re-scheduling the old request - triggers a CPU cache hit + load. - - GPU pool: 8 blocks (7 usable). _target_free = ceil(64/16) = 4. - """ - fix = make_scheduler(num_cpu_blocks=8, num_gpu_blocks=8, lazy=True) - sched = fix.scheduler - gpu_pool = fix.gpu_block_pool - - num_blocks = 2 - - # --- Step 1: Schedule req_old, compute, and finish --- - req_old = make_request(num_blocks=num_blocks) - gpu_blocks_old = _allocate_gpu_blocks(gpu_pool, req_old, num_blocks, group_id=0) - gpu_pool.free_blocks(gpu_blocks_old) - - # Allocate filler blocks so req_old's hashed blocks move to LRU head. - # 7 usable - 2 (req_old freed) = 5 other free blocks to consume. - fillers = _flush_old_blocks_to_lru_head(gpu_pool, num_filler_blocks=5) - - # --- Step 2: Lazy scanner should offload req_old's blocks --- - sched_out = make_scheduler_output({}) - meta = sched.build_connector_meta(sched_out) - assert meta.store_event >= 0, "Expected lazy store to offload old blocks" - assert len(meta.store_gpu_blocks) == num_blocks - simulate_store_completion(sched, meta.store_event) - - # Free fillers to restore pool capacity. - gpu_pool.free_blocks(fillers) - - # --- Step 3: Re-schedule req_old — should get CPU cache hit --- - req_old2 = Request( - request_id="req-old-reload", - prompt_token_ids=req_old.prompt_token_ids, - sampling_params=req_old.sampling_params, - pooling_params=None, - mm_features=None, - block_hasher=req_old._block_hasher, - ) - hit_tokens, is_async = sched.get_num_new_matched_tokens( - req_old2, num_computed_tokens=0 - ) - # make_request pads num_tokens by +1 beyond the last full block, so the - # manager's max_hit_len = num_tokens - 1 cap leaves all full blocks intact. - expected_hit = num_blocks * BLOCK_SIZE - assert hit_tokens == expected_hit, ( - f"Expected {expected_hit} hit tokens, got {hit_tokens}" - ) - assert is_async is True - - # Allocate fresh GPU blocks for the load. - gpu_blocks_load = gpu_pool.get_new_blocks(num_blocks) - kv_blocks_load = KVCacheBlocks(blocks=(gpu_blocks_load,)) - sched.update_state_after_alloc( - req_old2, kv_blocks_load, num_external_tokens=hit_tokens - ) - - sched_out2 = make_scheduler_output({req_old2.request_id: 1}) - meta2 = sched.build_connector_meta(sched_out2) - assert meta2.load_event >= 0, "Expected a load event to be assigned" - assert len(meta2.load_gpu_blocks) > 0 - - -# --------------------------------------------------------------------------- -# Test 2a: Eager duplicate store is skipped -# --------------------------------------------------------------------------- -def test_eager_duplicate_store_skipped() -> None: - """Eager: storing the same block hashes twice should not allocate new CPU blocks.""" - fix = make_scheduler(num_cpu_blocks=8, num_gpu_blocks=16, lazy=False) - sched = fix.scheduler - - num_blocks = 2 - req = make_request(num_blocks=num_blocks) - - kv_blocks = _alloc_and_register(fix, req, num_blocks) - sched.update_state_after_alloc(req, kv_blocks, num_external_tokens=0) - block_ids = kv_blocks.get_block_ids() - sched_out = make_scheduler_output( - {req.request_id: num_blocks * BLOCK_SIZE}, - new_reqs={req.request_id: block_ids}, - ) - - meta1 = sched.build_connector_meta(sched_out) - assert meta1.store_event >= 0 - simulate_store_completion(sched, meta1.store_event) - cpu_free_after_first = get_cpu_free_blocks(sched) - - # Second request with identical hashes — should skip store - req2 = Request( - request_id="req-dup-eager", - prompt_token_ids=req.prompt_token_ids, - sampling_params=req.sampling_params, - pooling_params=None, - mm_features=None, - block_hasher=req._block_hasher, - ) - kv_blocks2 = _alloc_and_register(fix, req2, num_blocks) - sched.update_state_after_alloc(req2, kv_blocks2, num_external_tokens=0) - block_ids2 = kv_blocks2.get_block_ids() - sched_out2 = make_scheduler_output( - {req2.request_id: num_blocks * BLOCK_SIZE}, - new_reqs={req2.request_id: block_ids2}, - ) - - meta2 = sched.build_connector_meta(sched_out2) - if meta2.store_event >= 0: - assert len(meta2.store_cpu_blocks) == 0, ( - "Expected no new CPU blocks for duplicate hashes" - ) - assert get_cpu_free_blocks(sched) == cpu_free_after_first - - -# --------------------------------------------------------------------------- -# Test 2b: Eager dedup of in-flight stores across consecutive steps -# --------------------------------------------------------------------------- -def test_eager_in_flight_store_dedup_across_steps() -> None: - """Eager: a second request sharing a prefix with an in-flight store - must not re-offload the same GPU blocks before completion lands. - - Simulates a GPU prefix-cache hit by reusing the first request's - GPU block IDs in the second scheduler step, which is the path the - real scheduler takes when two requests share a prefix. - """ - fix = make_scheduler(num_cpu_blocks=8, num_gpu_blocks=16, lazy=False) - sched = fix.scheduler - - num_blocks = 2 - req = make_request(num_blocks=num_blocks) - - kv_blocks = _alloc_and_register(fix, req, num_blocks) - sched.update_state_after_alloc(req, kv_blocks, num_external_tokens=0) - block_ids = kv_blocks.get_block_ids() - sched_out = make_scheduler_output( - {req.request_id: num_blocks * BLOCK_SIZE}, - new_reqs={req.request_id: block_ids}, - ) - - meta1 = sched.build_connector_meta(sched_out) - assert meta1.store_event >= 0 - assert len(meta1.store_cpu_blocks) == num_blocks - # In-flight set tracks the scheduled GPU blocks until completion. - assert sched._in_flight_store_gpu_blocks == set(meta1.store_gpu_blocks) - cpu_free_after_first = get_cpu_free_blocks(sched) - - # Second request shares the prefix and reuses the same GPU block IDs - # (the real scheduler path: GPU prefix cache returns the same blocks). - # Do NOT simulate completion — the first store is still in-flight. - req2 = Request( - request_id="req-dup-eager-inflight", - prompt_token_ids=req.prompt_token_ids, - sampling_params=req.sampling_params, - pooling_params=None, - mm_features=None, - block_hasher=req._block_hasher, - ) - req2.num_computed_tokens = num_blocks * BLOCK_SIZE - sched.update_state_after_alloc(req2, kv_blocks, num_external_tokens=0) - sched_out2 = make_scheduler_output( - {req2.request_id: num_blocks * BLOCK_SIZE}, - new_reqs={req2.request_id: block_ids}, - ) - - meta2 = sched.build_connector_meta(sched_out2) - if meta2.store_event >= 0: - assert len(meta2.store_cpu_blocks) == 0, ( - "Expected no new CPU blocks for in-flight duplicate hashes" - ) - assert get_cpu_free_blocks(sched) == cpu_free_after_first, ( - "Second request should not consume CPU blocks while the first " - "store is still in-flight" - ) - - # After completion, the in-flight set is cleared. - simulate_store_completion(sched, meta1.store_event) - assert sched._in_flight_store_gpu_blocks == set() - - -# --------------------------------------------------------------------------- -# Test 2c: Lazy duplicate store is skipped -# --------------------------------------------------------------------------- -def test_lazy_duplicate_store_skipped() -> None: - """Lazy: blocks already offloaded to CPU should not be offloaded again. - - Same pattern as the lazy roundtrip: flush old blocks to LRU head, offload, - then repeat with the same hashes and verify no new CPU allocation. - """ - fix = make_scheduler(num_cpu_blocks=8, num_gpu_blocks=8, lazy=True) - sched = fix.scheduler - gpu_pool = fix.gpu_block_pool - - num_blocks = 2 - req = make_request(num_blocks=num_blocks) - - # Schedule + finish → hashed blocks in free queue - gpu_blocks = _allocate_gpu_blocks(gpu_pool, req, num_blocks, group_id=0) - gpu_pool.free_blocks(gpu_blocks) - - # Flush old blocks to LRU head, then trigger lazy offload. - fillers = _flush_old_blocks_to_lru_head(gpu_pool, num_filler_blocks=5) - meta1 = sched.build_connector_meta(make_scheduler_output({})) - assert meta1.store_event >= 0 - simulate_store_completion(sched, meta1.store_event) - gpu_pool.free_blocks(fillers) - cpu_free_after_first = get_cpu_free_blocks(sched) - - # Allocate blocks with the same hashes and free them again. - # The scanner should see they are already in CPU cache and skip them. - req2 = Request( - request_id="req-dup-lazy", - prompt_token_ids=req.prompt_token_ids, - sampling_params=req.sampling_params, - pooling_params=None, - mm_features=None, - block_hasher=req._block_hasher, - ) - gpu_blocks2 = _allocate_gpu_blocks(gpu_pool, req2, num_blocks, group_id=0) - gpu_pool.free_blocks(gpu_blocks2) - - # Flush again so the hashed blocks are at LRU head for the scanner. - fillers2 = _flush_old_blocks_to_lru_head(gpu_pool, num_filler_blocks=5) - meta2 = sched.build_connector_meta(make_scheduler_output({})) - gpu_pool.free_blocks(fillers2) - - # Either no store event, or zero new CPU blocks (already cached). - if meta2.store_event >= 0: - assert len(meta2.store_cpu_blocks) == 0, ( - "Expected no new CPU blocks for duplicate hashes" - ) - assert get_cpu_free_blocks(sched) == cpu_free_after_first - - -# --------------------------------------------------------------------------- -# Test 3: LRU eviction order -# --------------------------------------------------------------------------- -def test_lru_eviction_order() -> None: - """With limited CPU space, oldest blocks should be evicted first. - - CPU block pool: num_cpu_blocks=5 -> 4 free usable blocks (1 taken by null_block). - After storing 4 blocks (2 req_a + 2 req_b), all free slots are occupied by - cached blocks (ref_cnt=0, in hash map). When 2 more are stored (req_c), - 2 LRU blocks from req_a get evicted from the cache to make room. - """ - # 5 total = 4 usable (null_block takes 1), filling exactly with 4 blocks - fix = make_scheduler(num_cpu_blocks=5, num_gpu_blocks=16, lazy=False) - sched = fix.scheduler - - # Fill CPU with 4 blocks: 2 requests x 2 blocks (in LRU insertion order) - req_a = make_request(num_blocks=2) - req_b = make_request(num_blocks=2) - - kv_a = _alloc_and_register(fix, req_a, 2) - kv_b = _alloc_and_register(fix, req_b, 2) - sched.update_state_after_alloc(req_a, kv_a, num_external_tokens=0) - sched.update_state_after_alloc(req_b, kv_b, num_external_tokens=0) - - ids_a = kv_a.get_block_ids() - ids_b = kv_b.get_block_ids() - sched_out = make_scheduler_output( - { - req_a.request_id: 2 * BLOCK_SIZE, - req_b.request_id: 2 * BLOCK_SIZE, - }, - new_reqs={ - req_a.request_id: ids_a, - req_b.request_id: ids_b, - }, - ) - meta = sched.build_connector_meta(sched_out) - assert meta.store_event >= 0 - simulate_store_completion(sched, meta.store_event) - - # Verify all 4 blocks are cached in CPU hash map - for i, bhash in enumerate(req_a.block_hashes[:2]): - bhash_with_group = make_block_hash_with_group_id(bhash, 0) - assert ( - sched.cpu_block_pool.cached_block_hash_to_block.get_one_block( - bhash_with_group - ) - is not None - ), f"req_a block {i} should be cached after store" - for i, bhash in enumerate(req_b.block_hashes[:2]): - bhash_with_group = make_block_hash_with_group_id(bhash, 0) - assert ( - sched.cpu_block_pool.cached_block_hash_to_block.get_one_block( - bhash_with_group - ) - is not None - ), f"req_b block {i} should be cached after store" - - # Store 2 more blocks from a new request - must evict 2 LRU blocks (req_a) - req_c = make_request(num_blocks=2) - kv_c = _alloc_and_register(fix, req_c, 2) - sched.update_state_after_alloc(req_c, kv_c, num_external_tokens=0) - - ids_c = kv_c.get_block_ids() - sched_out2 = make_scheduler_output( - {req_c.request_id: 2 * BLOCK_SIZE}, - new_reqs={req_c.request_id: ids_c}, - ) - meta2 = sched.build_connector_meta(sched_out2) - assert meta2.store_event >= 0 - simulate_store_completion(sched, meta2.store_event) - - # req_a hashes should be evicted from CPU (they were LRU) - for i, bhash in enumerate(req_a.block_hashes[:2]): - bhash_with_group = make_block_hash_with_group_id(bhash, 0) - cache_map = sched.cpu_block_pool.cached_block_hash_to_block - cached = cache_map.get_one_block(bhash_with_group) - assert cached is None, f"req_a block {i} should have been evicted" - - # req_b and req_c hashes should be present - for i, bhash in enumerate(req_b.block_hashes[:2]): - bhash_with_group = make_block_hash_with_group_id(bhash, 0) - cache_map = sched.cpu_block_pool.cached_block_hash_to_block - cached = cache_map.get_one_block(bhash_with_group) - assert cached is not None, f"req_b block {i} should still be cached" - - for i, bhash in enumerate(req_c.block_hashes[:2]): - bhash_with_group = make_block_hash_with_group_id(bhash, 0) - cache_map = sched.cpu_block_pool.cached_block_hash_to_block - cached = cache_map.get_one_block(bhash_with_group) - assert cached is not None, f"req_c block {i} should still be cached" - - -# --------------------------------------------------------------------------- -# Test 4: Touched blocks survive eviction -# --------------------------------------------------------------------------- -def test_touched_blocks_survive_eviction() -> None: - """Touching CPU blocks updates their LRU position, protecting them from eviction.""" - # 5 total = 4 usable (null_block takes 1) - fix = make_scheduler(num_cpu_blocks=5, num_gpu_blocks=16, lazy=False) - sched = fix.scheduler - - # Fill CPU with 4 blocks (req_a: 2, req_b: 2) in LRU order - req_a = make_request(num_blocks=2) - req_b = make_request(num_blocks=2) - - kv_a = _alloc_and_register(fix, req_a, 2) - kv_b = _alloc_and_register(fix, req_b, 2) - sched.update_state_after_alloc(req_a, kv_a, num_external_tokens=0) - sched.update_state_after_alloc(req_b, kv_b, num_external_tokens=0) - - ids_a = kv_a.get_block_ids() - ids_b = kv_b.get_block_ids() - sched_out = make_scheduler_output( - { - req_a.request_id: 2 * BLOCK_SIZE, - req_b.request_id: 2 * BLOCK_SIZE, - }, - new_reqs={ - req_a.request_id: ids_a, - req_b.request_id: ids_b, - }, - ) - meta = sched.build_connector_meta(sched_out) - simulate_store_completion(sched, meta.store_event) - - # Touch req_a's CPU blocks to make them most-recently-used - cpu_pool = sched.cpu_block_pool - for bhash in req_a.block_hashes[:2]: - bhash_with_group = make_block_hash_with_group_id(bhash, 0) - cached_blk = cpu_pool.cached_block_hash_to_block.get_one_block(bhash_with_group) - assert cached_blk is not None - cpu_pool.touch([cached_blk]) - # Undo touch to return ref_cnt to 0 - # (so it's a free candidate but at MRU position) - cpu_pool.free_blocks([cached_blk]) - - # Now store 2 more blocks; req_b (LRU front) should be evicted, not req_a - req_c = make_request(num_blocks=2) - kv_c = _alloc_and_register(fix, req_c, 2) - sched.update_state_after_alloc(req_c, kv_c, num_external_tokens=0) - - ids_c = kv_c.get_block_ids() - sched_out2 = make_scheduler_output( - {req_c.request_id: 2 * BLOCK_SIZE}, - new_reqs={req_c.request_id: ids_c}, - ) - meta2 = sched.build_connector_meta(sched_out2) - simulate_store_completion(sched, meta2.store_event) - - # req_b should be evicted (LRU), req_a and req_c should survive - for i, bhash in enumerate(req_b.block_hashes[:2]): - bhash_with_group = make_block_hash_with_group_id(bhash, 0) - cached = cpu_pool.cached_block_hash_to_block.get_one_block(bhash_with_group) - assert cached is None, f"req_b block {i} should have been evicted (it was LRU)" - - for i, bhash in enumerate(req_a.block_hashes[:2]): - bhash_with_group = make_block_hash_with_group_id(bhash, 0) - cached = cpu_pool.cached_block_hash_to_block.get_one_block(bhash_with_group) - assert cached is not None, f"req_a block {i} should survive (was touched/MRU)" - - -# --------------------------------------------------------------------------- -# Test 5: Preemption no CPU block leak -# --------------------------------------------------------------------------- -def test_preemption_no_cpu_block_leak() -> None: - """request_finished during in-flight load defers cleanup; - completes after load done.""" - fix = make_scheduler(num_cpu_blocks=8, num_gpu_blocks=16, lazy=False) - sched = fix.scheduler - - num_blocks = 2 - - # First: store blocks to CPU - req = make_request(num_blocks=num_blocks) - kv_blocks = _alloc_and_register(fix, req, num_blocks) - sched.update_state_after_alloc(req, kv_blocks, num_external_tokens=0) - block_ids = kv_blocks.get_block_ids() - sched_out = make_scheduler_output( - {req.request_id: num_blocks * BLOCK_SIZE}, - new_reqs={req.request_id: block_ids}, - ) - meta = sched.build_connector_meta(sched_out) - simulate_store_completion(sched, meta.store_event) - - # Create new request with same tokens, check hit - req2 = Request( - request_id="req-preempt-load", - prompt_token_ids=req.prompt_token_ids, - sampling_params=req.sampling_params, - pooling_params=None, - mm_features=None, - block_hasher=req._block_hasher, - ) - hit_tokens, is_async = sched.get_num_new_matched_tokens(req2, num_computed_tokens=0) - assert hit_tokens is not None and hit_tokens > 0 - - gpu_blocks2 = fix.gpu_block_pool.get_new_blocks(num_blocks) - kv_blocks2 = KVCacheBlocks(blocks=(gpu_blocks2,)) - sched.update_state_after_alloc(req2, kv_blocks2, num_external_tokens=hit_tokens) - - # Assign load_event via build_connector_meta - block_ids2 = kv_blocks2.get_block_ids() - sched_out2 = make_scheduler_output( - {req2.request_id: 1}, - new_reqs={req2.request_id: block_ids2}, - ) - meta2 = sched.build_connector_meta(sched_out2) - assert meta2.load_event >= 0 - - # Request finishes BEFORE load completes -> deferred - sched.request_finished(req2, block_ids=[]) - assert req2.request_id in sched._reqs_to_load - assert sched._reqs_to_load[req2.request_id].finished is True - - # Now simulate load completion -> cleanup fires - simulate_load_completion(sched, {req2.request_id}) - assert req2.request_id not in sched._reqs_to_load - - -# --------------------------------------------------------------------------- -# Test 6: Eager store preemption cleanup -# --------------------------------------------------------------------------- -def test_eager_store_preemption_cleanup() -> None: - """In eager mode, finishing a request during in-flight store defers cleanup.""" - fix = make_scheduler(num_cpu_blocks=8, num_gpu_blocks=16, lazy=False) - sched = fix.scheduler - - num_blocks = 2 - req = make_request(num_blocks=num_blocks) - kv_blocks = _alloc_and_register(fix, req, num_blocks) - sched.update_state_after_alloc(req, kv_blocks, num_external_tokens=0) - - block_ids = kv_blocks.get_block_ids() - sched_out = make_scheduler_output( - {req.request_id: num_blocks * BLOCK_SIZE}, - new_reqs={req.request_id: block_ids}, - ) - meta = sched.build_connector_meta(sched_out) - store_event = meta.store_event - assert store_event >= 0 - - # The request gets store_events populated - assert req.request_id in sched._reqs_to_store - store_state = sched._reqs_to_store[req.request_id] - assert store_event in store_state.store_events - - # Finish request while store still in-flight -> deferred - sched.request_finished(req, block_ids=[]) - assert req.request_id in sched._reqs_to_store - assert sched._reqs_to_store[req.request_id].finished is True - - # Simulate store completion -> deferred cleanup fires - simulate_store_completion(sched, store_event) - assert req.request_id not in sched._reqs_to_store - - -# --------------------------------------------------------------------------- -# Test 7: In-flight finish deferred cleanup (load variant) -# --------------------------------------------------------------------------- -def test_inflight_finish_deferred_cleanup() -> None: - """Store, then start a load, request_finished defers, - load completion fires cleanup.""" - fix = make_scheduler(num_cpu_blocks=8, num_gpu_blocks=16, lazy=False) - sched = fix.scheduler - - num_blocks = 2 - - # Store - req = make_request(num_blocks=num_blocks) - kv_blocks = _alloc_and_register(fix, req, num_blocks) - sched.update_state_after_alloc(req, kv_blocks, num_external_tokens=0) - block_ids = kv_blocks.get_block_ids() - sched_out = make_scheduler_output( - {req.request_id: num_blocks * BLOCK_SIZE}, - new_reqs={req.request_id: block_ids}, - ) - meta = sched.build_connector_meta(sched_out) - simulate_store_completion(sched, meta.store_event) - - # Load - req2 = Request( - request_id="req-inflight-load", - prompt_token_ids=req.prompt_token_ids, - sampling_params=req.sampling_params, - pooling_params=None, - mm_features=None, - block_hasher=req._block_hasher, - ) - hit_tokens, _ = sched.get_num_new_matched_tokens(req2, num_computed_tokens=0) - assert hit_tokens is not None and hit_tokens > 0 - - gpu_blocks2 = fix.gpu_block_pool.get_new_blocks(num_blocks) - kv_blocks2 = KVCacheBlocks(blocks=(gpu_blocks2,)) - sched.update_state_after_alloc(req2, kv_blocks2, num_external_tokens=hit_tokens) - - block_ids2 = kv_blocks2.get_block_ids() - sched_out2 = make_scheduler_output( - {req2.request_id: 1}, - new_reqs={req2.request_id: block_ids2}, - ) - meta2 = sched.build_connector_meta(sched_out2) - assert meta2.load_event >= 0 - - # Finish before load completes - sched.request_finished(req2, block_ids=[]) - assert req2.request_id in sched._reqs_to_load - - # Simulate load completion -> request removed - simulate_load_completion(sched, {req2.request_id}) - assert req2.request_id not in sched._reqs_to_load - - -# --------------------------------------------------------------------------- -# Test 8: Null GPU blocks are skipped in store and load transfer pairs -# --------------------------------------------------------------------------- -def test_multi_group_null_blocks_skipped() -> None: - """The null block id never appears in store or load pairs. - - A null block-table slot whose hash is still cached on the GPU is stored - from the cached block instead; the null block itself is never copied. - """ - fix = make_scheduler(num_cpu_blocks=8, num_gpu_blocks=16, num_groups=1, lazy=False) - sched = fix.scheduler - gpu_pool = fix.gpu_block_pool - - num_blocks = 2 - req = make_request(num_blocks=num_blocks) - - # Allocate real blocks (with hashes) and use the null_block as a placeholder - gpu_blocks = _allocate_gpu_blocks(gpu_pool, req, num_blocks, group_id=0) - null_block = gpu_pool.null_block - - # Mix: [real_block, null_block] — null_block has no hash, should be skipped - mixed_blocks = [gpu_blocks[0], null_block] - kv_blocks = KVCacheBlocks(blocks=(mixed_blocks,)) - req.num_computed_tokens = num_blocks * BLOCK_SIZE - sched.update_state_after_alloc(req, kv_blocks, num_external_tokens=0) - - block_ids = kv_blocks.get_block_ids() - sched_out = make_scheduler_output( - {req.request_id: num_blocks * BLOCK_SIZE}, - new_reqs={req.request_id: block_ids}, - ) - meta = sched.build_connector_meta(sched_out) - - # Null block's ID should NOT appear in store_gpu_blocks - null_block_id = null_block.block_id - assert null_block_id not in meta.store_gpu_blocks, ( - f"Null block id {null_block_id} should not appear in store transfer pairs" - ) - - # The nulled slot's hash is still cached on the GPU, so that block is - # recovered from the pool and stored alongside the real one. - assert len(meta.store_gpu_blocks) == 2 - assert gpu_blocks[0].block_id in meta.store_gpu_blocks - assert gpu_blocks[1].block_id in meta.store_gpu_blocks - - # Complete the store - assert meta.store_event >= 0 - simulate_store_completion(sched, meta.store_event) - - # Create matching request and get load hit - req2 = Request( - request_id="req-null-load", - prompt_token_ids=req.prompt_token_ids, - sampling_params=req.sampling_params, - pooling_params=None, - mm_features=None, - block_hasher=req._block_hasher, - ) - hit_tokens, is_async = sched.get_num_new_matched_tokens(req2, num_computed_tokens=0) - assert hit_tokens == 2 * BLOCK_SIZE - assert is_async is True - - # Allocate new GPU blocks for the load - gpu_blocks2 = gpu_pool.get_new_blocks(2) - kv_blocks2 = KVCacheBlocks(blocks=(gpu_blocks2,)) - sched.update_state_after_alloc(req2, kv_blocks2, num_external_tokens=hit_tokens) - - sched_out2 = make_scheduler_output({req2.request_id: 1}) - meta2 = sched.build_connector_meta(sched_out2) - - # Null block's ID should NOT appear in load_gpu_blocks - assert null_block_id not in meta2.load_gpu_blocks, ( - f"Null block id {null_block_id} should not appear in load transfer pairs" - ) - - -# --------------------------------------------------------------------------- -# Test 8b: Non-prefix-cacheable scratch groups take no part in store or load -# --------------------------------------------------------------------------- -def test_scratch_group_excluded_from_store_and_load() -> None: - """A scratch group (GLM-5.3-Flash kpool tail) is never stored or loaded. - - Its single per-request block carries no hash, so the store must offload - only the attention blocks and the load must pair only attention blocks. - """ - fix = make_scheduler( - num_cpu_blocks=8, - num_gpu_blocks=16, - kv_cache_config=_make_scratch_kv_cache_config(16), - ) - sched = fix.scheduler - gpu_pool = fix.gpu_block_pool - num_blocks = 2 - - req = make_request(num_blocks=num_blocks) - fa_blocks = _allocate_gpu_blocks(gpu_pool, req, num_blocks, group_id=0) - scratch_block = gpu_pool.get_new_blocks(1) - kv_blocks = KVCacheBlocks(blocks=(fa_blocks, scratch_block)) - req.num_computed_tokens = num_blocks * BLOCK_SIZE - sched.update_state_after_alloc(req, kv_blocks, num_external_tokens=0) - sched_out = make_scheduler_output( - {req.request_id: num_blocks * BLOCK_SIZE}, - new_reqs={req.request_id: kv_blocks.get_block_ids()}, - ) - meta = sched.build_connector_meta(sched_out) - assert sorted(meta.store_gpu_blocks) == sorted(b.block_id for b in fa_blocks) - simulate_store_completion(sched, meta.store_event) - - req2 = Request( - request_id="req-scratch-load", - prompt_token_ids=req.prompt_token_ids, - sampling_params=req.sampling_params, - pooling_params=None, - mm_features=None, - block_hasher=req._block_hasher, - ) - hit_tokens, is_async = sched.get_num_new_matched_tokens(req2, num_computed_tokens=0) - assert hit_tokens == num_blocks * BLOCK_SIZE - assert is_async is True - - fa_blocks2 = gpu_pool.get_new_blocks(num_blocks) - scratch_block2 = gpu_pool.get_new_blocks(1) - kv_blocks2 = KVCacheBlocks(blocks=(fa_blocks2, scratch_block2)) - sched.update_state_after_alloc(req2, kv_blocks2, num_external_tokens=hit_tokens) - sched_out2 = make_scheduler_output( - {req2.request_id: 1}, - new_reqs={req2.request_id: kv_blocks2.get_block_ids()}, - ) - meta2 = sched.build_connector_meta(sched_out2) - assert sorted(meta2.load_gpu_blocks) == sorted(b.block_id for b in fa_blocks2) - assert len(meta2.load_cpu_blocks) == num_blocks - - -# --------------------------------------------------------------------------- -# Test 9: Chunked prefill accumulates block_ids across steps -# --------------------------------------------------------------------------- -def test_chunked_prefill_reads_live_block_ids() -> None: - """With chunked prefill, block IDs accumulate across scheduler steps. - _prepare_eager_store_specs reads block IDs from scheduler_output via - yield_req_data, so the store should reflect the updated (larger) block - list, not a stale snapshot.""" - fix = make_scheduler(num_cpu_blocks=8, num_gpu_blocks=16, lazy=False) - sched = fix.scheduler - - num_blocks = 4 - req = make_request(num_blocks=num_blocks) - - # First chunk: allocate 2 blocks - kv_blocks_first = _alloc_and_register(fix, req, 2) - sched.update_state_after_alloc(req, kv_blocks_first, num_external_tokens=0) - - assert req.request_id in sched._reqs_to_store - # Should still be exactly 1 entry in _reqs_to_store - assert list(sched._reqs_to_store.keys()).count(req.request_id) == 1 - - # Build connector meta with 2 blocks — stores the first 2 - ids_first = kv_blocks_first.get_block_ids() - sched_out1 = make_scheduler_output( - {req.request_id: 2 * BLOCK_SIZE}, - new_reqs={req.request_id: ids_first}, - ) - meta1 = sched.build_connector_meta(sched_out1) - assert meta1.store_event >= 0 - assert len(meta1.store_gpu_blocks) == 2 - simulate_store_completion(sched, meta1.store_event) - - # Second chunk: allocate 4 blocks total (2 new ones) - kv_blocks_second = _alloc_and_register(fix, req, num_blocks) - # update_state_after_alloc is idempotent for store registration - sched.update_state_after_alloc(req, kv_blocks_second, num_external_tokens=0) - - # Still exactly 1 entry - assert list(sched._reqs_to_store.keys()).count(req.request_id) == 1 - - # The second chunk's NEW block IDs (positions 2,3) are passed as - # cached_req_new_blocks. The full block_ids include both old and new, - # but yield_req_data only appends the new_block_ids for cached reqs. - ids_second_full = kv_blocks_second.get_block_ids() - # New blocks are those beyond the first chunk - new_block_ids = tuple(ids_second_full[g][2:] for g in range(len(ids_second_full))) - sched_out2 = make_scheduler_output( - {req.request_id: 2 * BLOCK_SIZE}, - cached_req_new_blocks={req.request_id: new_block_ids}, - ) - meta2 = sched.build_connector_meta(sched_out2) - assert meta2.store_event >= 0 - # Only the 2 NEW blocks should be stored (first 2 already done) - assert len(meta2.store_gpu_blocks) == 2 - - -@pytest.mark.parametrize("use_v2_model_runner", [False, True]) -@pytest.mark.parametrize("same_output", [False, True]) -def test_eager_store_resets_block_cursor_after_preemption( - use_v2_model_runner: bool, - same_output: bool, -) -> None: - """A resumed request stores its confirmed tail after preemption.""" - fix = make_scheduler(num_cpu_blocks=8, num_gpu_blocks=8, lazy=False) - sched = fix.scheduler - req = make_request(num_blocks=3) - - initial_blocks = _alloc_and_register(fix, req, num_blocks=2) - sched.update_state_after_alloc(req, initial_blocks, num_external_tokens=0) - initial_output = make_scheduler_output( - {req.request_id: 2 * BLOCK_SIZE}, - new_reqs={req.request_id: initial_blocks.get_block_ids()}, - ) - initial_meta = sched.build_connector_meta(initial_output) - assert len(initial_meta.store_gpu_blocks) == 2 - simulate_store_completion(sched, initial_meta.store_event) - - if not same_output: - preemption_output = make_scheduler_output({}) - preemption_output.preempted_req_ids = {req.request_id} - sched.build_connector_meta(preemption_output) - fix.gpu_block_pool.free_blocks(initial_blocks.blocks[0]) - - resumed_blocks = _alloc_and_register(fix, req, num_blocks=3) - sched.update_state_after_alloc(req, resumed_blocks, num_external_tokens=0) - if use_v2_model_runner: - resumed_output = make_scheduler_output( - {req.request_id: 3 * BLOCK_SIZE}, - new_reqs={req.request_id: resumed_blocks.get_block_ids()}, - ) - else: - resumed_output = make_scheduler_output( - {req.request_id: 3 * BLOCK_SIZE}, - cached_req_new_blocks={req.request_id: resumed_blocks.get_block_ids()}, - ) - resumed_output.scheduled_cached_reqs.resumed_req_ids.add(req.request_id) - if same_output: - resumed_output.preempted_req_ids = {req.request_id} - - resumed_meta = sched.build_connector_meta(resumed_output) - assert resumed_meta.store_gpu_blocks == [resumed_blocks.blocks[0][2].block_id] - - -# --------------------------------------------------------------------------- -# Test 10: Partial GPU prefix hit + CPU load + new compute blocks -# --------------------------------------------------------------------------- -def test_partial_gpu_prefix_plus_cpu_load() -> None: - """When GPU has a prefix cache hit for the first N blocks, CPU has a - hit for the next M blocks, and there are P new blocks needing fresh - compute, the block layout is: - - | comp (N) | ext_comp (M) | new (P) | - - External blocks sit in the middle — not at the beginning or end. - The load path must target hashes at positions [N, N+M). - - Request: 6 blocks (0..5). - - Store all 6 to CPU. - - New request: GPU prefix cache hits blocks 0,1 (hashed). - CPU hits blocks 2,3. Blocks 4,5 are new (need compute). - - update_state_after_alloc receives 6 GPU blocks: - [0,1] hashed (comp), [2,3] unhashed (ext_comp), [4,5] unhashed (new). - - Load must target hash positions 2,3. - """ - fix = make_scheduler(num_cpu_blocks=8, num_gpu_blocks=16, lazy=False) - sched = fix.scheduler - gpu_pool = fix.gpu_block_pool - - num_blocks = 6 - req = make_request(num_blocks=num_blocks) - - # Store all 6 blocks to CPU via eager store. - kv_blocks = _alloc_and_register(fix, req, num_blocks) - sched.update_state_after_alloc(req, kv_blocks, num_external_tokens=0) - block_ids = kv_blocks.get_block_ids() - sched_out = make_scheduler_output( - {req.request_id: num_blocks * BLOCK_SIZE}, - new_reqs={req.request_id: block_ids}, - ) - meta = sched.build_connector_meta(sched_out) - assert meta.store_event >= 0 - simulate_store_completion(sched, meta.store_event) - - # New request with same tokens — but only partial GPU prefix hit. - req2 = Request( - request_id="req-partial-gpu", - prompt_token_ids=req.prompt_token_ids, - sampling_params=req.sampling_params, - pooling_params=None, - mm_features=None, - block_hasher=req._block_hasher, - ) - - # GPU prefix cache hits the first 2 blocks. - gpu_local_computed = 2 * BLOCK_SIZE - hit_tokens, is_async = sched.get_num_new_matched_tokens( - req2, num_computed_tokens=gpu_local_computed - ) - # CPU has all 6 blocks stored. make_request pads num_tokens by +1, so - # the manager's num_tokens - 1 cap leaves all full blocks intact: - # remaining hashable range = 6 - 2 = 4 blocks, all hit. - num_cpu_hit_blocks = 4 - assert hit_tokens == num_cpu_hit_blocks * BLOCK_SIZE, ( - f"Expected {num_cpu_hit_blocks * BLOCK_SIZE} CPU hit tokens, got {hit_tokens}" - ) - assert is_async is True - - # Simulate what the real scheduler does: only accept 2 of the 4 CPU hit - # blocks as external (e.g. due to budget constraints), leaving 2 new - # blocks for fresh compute. - num_ext_blocks = 2 - num_new_blocks = 2 - external_tokens = num_ext_blocks * BLOCK_SIZE - - # Build block list matching real layout: | comp(2) | ext_comp(2) | new(2) | - # comp: GPU prefix cache hit — blocks with hashes - gpu_comp = _allocate_gpu_blocks(gpu_pool, req2, 2, group_id=0) - # ext_comp + new: freshly allocated, no hashes - gpu_ext_and_new = gpu_pool.get_new_blocks(num_ext_blocks + num_new_blocks) - all_gpu_blocks = gpu_comp + gpu_ext_and_new - kv_blocks2 = KVCacheBlocks(blocks=(all_gpu_blocks,)) - - # Critical call: with 2 hashed comp blocks and 2 external tokens worth - # of blocks, the manager must derive skipped=2 and load hashes [2,3]. - sched.update_state_after_alloc( - req2, kv_blocks2, num_external_tokens=external_tokens - ) - - block_ids2 = kv_blocks2.get_block_ids() - sched_out2 = make_scheduler_output( - {req2.request_id: num_new_blocks * BLOCK_SIZE}, - new_reqs={req2.request_id: block_ids2}, - ) - meta2 = sched.build_connector_meta(sched_out2) - assert meta2.load_event >= 0, "Expected a load event for partial GPU + CPU hit" - assert len(meta2.load_gpu_blocks) == num_ext_blocks - assert len(meta2.load_cpu_blocks) == num_ext_blocks - - # Verify the load targets the ext_comp GPU blocks (positions 2,3), - # not the comp blocks (0,1) or new blocks (4,5). - ext_block_ids = [b.block_id for b in gpu_ext_and_new[:num_ext_blocks]] - for bid in meta2.load_gpu_blocks: - assert bid in ext_block_ids, ( - f"Load GPU block {bid} should be an ext_comp block, not a comp or new block" - ) - - -# --------------------------------------------------------------------------- -# Test 11: TOCTOU between Phase A and Phase B (regression for #39702) -# --------------------------------------------------------------------------- -def test_toctou_cpu_hit_evicted_between_phases_no_crash() -> None: - """Regression for vllm-project/vllm#39702. - - When ``get_num_new_matched_tokens`` (Phase A) reports a CPU cache hit - of ``N`` tokens but ``update_state_after_alloc`` (Phase B) runs after - other requests have caused LRU eviction of those exact blocks, the - second ``find_longest_cache_hit`` call returns 0 while - ``num_external_tokens`` is still ``N``, triggering - ``AssertionError: Expected N hit tokens, got 0``. - - Setup: ``num_cpu_blocks=5`` (4 usable; null_block takes 1). - 1. Store req_a's 2 blocks to CPU. CPU: [a0, a1, _, _]. - 2. Phase A on req_b (same prompt as req_a) reports a 2-block hit. - Without the fix this does NOT pin a0/a1 — they remain at LRU front. - 3. Store req_c (2) + req_d (2) — 4 more blocks into 4 slots. With - a0/a1 unpinned and at LRU front, they get evicted. CPU: [c0, c1, - d0, d1]. - 4. Phase B on req_b: re-searches the CPU coordinator, finds 0 - cached hashes, asserts. - - After the fix, Phase A pins the hit blocks so step 3 evicts req_c's - blocks instead, and Phase B reads the cached ``(cpu_hit_blocks, - hit_length)`` tuple without re-searching. - """ - # 5 total = 4 usable (null_block takes 1) - fix = make_scheduler(num_cpu_blocks=5, num_gpu_blocks=16, lazy=False) - sched = fix.scheduler - - # --- Step 1: Store req_a's 2 blocks to CPU cache --- - req_a = make_request(num_blocks=2) - kv_a = _alloc_and_register(fix, req_a, 2) - sched.update_state_after_alloc(req_a, kv_a, num_external_tokens=0) - sched_out_a = make_scheduler_output( - {req_a.request_id: 2 * BLOCK_SIZE}, - new_reqs={req_a.request_id: kv_a.get_block_ids()}, - ) - meta_a = sched.build_connector_meta(sched_out_a) - assert meta_a.store_event >= 0 - simulate_store_completion(sched, meta_a.store_event) - - # --- Step 2: Phase A — req_b (same prompt as req_a) reports a CPU hit --- - req_b = Request( - request_id="req-b-toctou", - prompt_token_ids=req_a.prompt_token_ids, - sampling_params=req_a.sampling_params, - pooling_params=None, - mm_features=None, - block_hasher=req_a._block_hasher, - ) - hit_tokens, is_async = sched.get_num_new_matched_tokens( - req_b, num_computed_tokens=0 - ) - assert hit_tokens == 2 * BLOCK_SIZE, ( - f"Phase A should report 2 blocks of CPU hit, got {hit_tokens}" - ) - assert is_async is True - - # --- Step 3: TOCTOU window — fill CPU cache so LRU evicts req_a's blocks - # (in production this corresponds to other concurrent requests landing - # between Phase A and Phase B for req_b). 4 usable slots, req_a occupies 2; - # req_c (2) + req_d (2) require evicting 2 LRU blocks. --- - req_c = make_request(num_blocks=2) - req_d = make_request(num_blocks=2) - kv_c = _alloc_and_register(fix, req_c, 2) - kv_d = _alloc_and_register(fix, req_d, 2) - sched.update_state_after_alloc(req_c, kv_c, num_external_tokens=0) - sched.update_state_after_alloc(req_d, kv_d, num_external_tokens=0) - sched_out_pressure = make_scheduler_output( - { - req_c.request_id: 2 * BLOCK_SIZE, - req_d.request_id: 2 * BLOCK_SIZE, - }, - new_reqs={ - req_c.request_id: kv_c.get_block_ids(), - req_d.request_id: kv_d.get_block_ids(), - }, - ) - meta_pressure = sched.build_connector_meta(sched_out_pressure) - assert meta_pressure.store_event >= 0 - simulate_store_completion(sched, meta_pressure.store_event) - - # --- Step 4: Phase B — must not crash --- - # Before fix: AssertionError: Expected 32 hit tokens, got 0 - # After fix: Phase A pinned the hits → Step 3 evicted req_c instead, - # Phase B consumes the cached (cpu_hit_blocks, hit_length) tuple. - gpu_blocks_b = fix.gpu_block_pool.get_new_blocks(2) - kv_blocks_b = KVCacheBlocks(blocks=(gpu_blocks_b,)) - sched.update_state_after_alloc(req_b, kv_blocks_b, num_external_tokens=hit_tokens) - - # --- Step 5: with the fix, the load is queued correctly --- - sched_out_b = make_scheduler_output( - {req_b.request_id: 1}, - new_reqs={req_b.request_id: kv_blocks_b.get_block_ids()}, - ) - meta_b = sched.build_connector_meta(sched_out_b) - assert meta_b.load_event >= 0, ( - "Phase B should queue a load event using the pinned CPU hit blocks" - ) - assert len(meta_b.load_gpu_blocks) == 2 - assert len(meta_b.load_cpu_blocks) == 2 - - -# --------------------------------------------------------------------------- -# Test 12: Reset with pending eager stores waits for completion -# --------------------------------------------------------------------------- -def test_reset_pending_eager_stores() -> None: - """Eager mode: reset() abandons in-flight stores until they complete.""" - fix = make_scheduler(num_cpu_blocks=8, num_gpu_blocks=16, lazy=False) - sched = fix.scheduler - gpu_pool = fix.gpu_block_pool - - num_blocks = 2 - req = make_request(num_blocks=num_blocks) - - kv_blocks = _alloc_and_register(fix, req, num_blocks) - sched.update_state_after_alloc(req, kv_blocks, num_external_tokens=0) - block_ids = kv_blocks.get_block_ids() - sched_out = make_scheduler_output( - {req.request_id: num_blocks * BLOCK_SIZE}, - new_reqs={req.request_id: block_ids}, - ) - - meta = sched.build_connector_meta(sched_out) - assert meta.store_event >= 0 - assert len(sched._store_event_to_blocks) > 0 - - # GPU blocks should have elevated ref_cnt from touch() - for bid in meta.store_gpu_blocks: - assert gpu_pool.blocks[bid].ref_cnt > 0 - - # Free the request's own block refs (simulates preemption) - gpu_pool.free_blocks(gpu_pool.blocks[bid] for bid in block_ids[0]) - - # Reset should keep DMA refs pinned until the worker reports completion. - assert sched.reset() is False - assert len(sched._store_event_to_blocks) == 0 - assert len(sched._abandoned_store_event_to_blocks) == 1 - assert len(sched._reqs_to_store) == 0 - assert len(sched._store_event_to_reqs) == 0 - - num_used = gpu_pool.num_gpu_blocks - gpu_pool.get_num_free_blocks() - assert num_used > 1 - - simulate_store_completion(sched, meta.store_event) - assert len(sched._abandoned_store_event_to_blocks) == 0 - - # All GPU blocks should now be free (ref_cnt == 0) except null block - num_used = gpu_pool.num_gpu_blocks - gpu_pool.get_num_free_blocks() - assert num_used == 1, f"Expected only null block in use, got {num_used}" - - # GPU prefix cache reset should now succeed - assert gpu_pool.reset_prefix_cache() is True - assert sched.reset() is True - - -# --------------------------------------------------------------------------- -# Test 13: Reset with pending lazy stores waits for completion -# --------------------------------------------------------------------------- -def test_reset_pending_lazy_stores() -> None: - """Lazy mode: reset() abandons in-flight stores until they complete.""" - fix = make_scheduler(num_cpu_blocks=8, num_gpu_blocks=8, lazy=True) - sched = fix.scheduler - gpu_pool = fix.gpu_block_pool - - num_blocks = 2 - req = make_request(num_blocks=num_blocks) - - # Allocate, hash, and free — blocks move to free queue with hashes - gpu_blocks = _allocate_gpu_blocks(gpu_pool, req, num_blocks, group_id=0) - gpu_pool.free_blocks(gpu_blocks) - - # Push hashed blocks to LRU head - fillers = _flush_old_blocks_to_lru_head(gpu_pool, num_filler_blocks=5) - - # Lazy scanner offloads old hashed blocks - sched_out = make_scheduler_output({}) - meta = sched.build_connector_meta(sched_out) - assert meta.store_event >= 0 - assert len(sched._store_event_to_blocks) > 0 - - gpu_pool.free_blocks(fillers) - - # Reset should keep DMA refs pinned until the worker reports completion. - assert sched.reset() is False - assert len(sched._store_event_to_blocks) == 0 - assert len(sched._abandoned_store_event_to_blocks) == 1 - assert sched._cursor is None - - simulate_store_completion(sched, meta.store_event) - assert len(sched._abandoned_store_event_to_blocks) == 0 - assert sched.reset() is True - - # No CPU cache hits after reset - req2 = Request( - request_id="req-after-lazy-reset", - prompt_token_ids=req.prompt_token_ids, - sampling_params=req.sampling_params, - pooling_params=None, - mm_features=None, - block_hasher=req._block_hasher, - ) - hit_tokens, _ = sched.get_num_new_matched_tokens(req2, num_computed_tokens=0) - assert hit_tokens == 0, "CPU cache should be empty after reset" - - -@pytest.mark.parametrize("num_groups", [1, 2]) -@pytest.mark.parametrize("lazy", [False, True]) -def test_reset_releases_pending_cpu_hits(num_groups: int, lazy: bool) -> None: - """A lookup awaiting GPU allocation must not prevent a cache reset.""" - fix = make_scheduler(num_groups=num_groups, lazy=lazy) - sched = fix.scheduler - req = make_request(num_blocks=2) - cpu_blocks = [] - for group_id in range(num_groups): - cpu_blocks.extend( - _allocate_gpu_blocks(sched.cpu_block_pool, req, 2, group_id=group_id) - ) - sched.cpu_block_pool.free_blocks(cpu_blocks) - - assert sched.get_num_new_matched_tokens(req, 0) == (2 * BLOCK_SIZE, True) - assert all(block.ref_cnt == 1 for block in cpu_blocks) - assert not sched._reqs_to_load - - assert sched.reset() - assert all(block.ref_cnt == 0 for block in cpu_blocks) - assert not sched._pending_cpu_hits - assert sched.get_num_new_matched_tokens(req, 0) == (0, False) - assert sched.reset() - - -# --------------------------------------------------------------------------- -# Test 14: Reset with pending loads waits for completion -# --------------------------------------------------------------------------- -def test_reset_pending_loads() -> None: - """reset() abandons in-flight loads until they complete.""" - fix = make_scheduler(num_cpu_blocks=8, num_gpu_blocks=16, lazy=False) - sched = fix.scheduler - gpu_pool = fix.gpu_block_pool - - num_blocks = 2 - - # First store blocks to CPU - req = make_request(num_blocks=num_blocks) - kv_blocks = _alloc_and_register(fix, req, num_blocks) - sched.update_state_after_alloc(req, kv_blocks, num_external_tokens=0) - block_ids = kv_blocks.get_block_ids() - sched_out = make_scheduler_output( - {req.request_id: num_blocks * BLOCK_SIZE}, - new_reqs={req.request_id: block_ids}, - ) - meta = sched.build_connector_meta(sched_out) - simulate_store_completion(sched, meta.store_event) - - # Start a load — CPU cache hit - req2 = Request( - request_id="req-load-reset", - prompt_token_ids=req.prompt_token_ids, - sampling_params=req.sampling_params, - pooling_params=None, - mm_features=None, - block_hasher=req._block_hasher, - ) - hit_tokens, is_async = sched.get_num_new_matched_tokens(req2, num_computed_tokens=0) - assert hit_tokens is not None and hit_tokens > 0 - - gpu_blocks2 = gpu_pool.get_new_blocks(num_blocks) - kv_blocks2 = KVCacheBlocks(blocks=(gpu_blocks2,)) - sched.update_state_after_alloc(req2, kv_blocks2, num_external_tokens=hit_tokens) - - block_ids2 = kv_blocks2.get_block_ids() - sched_out2 = make_scheduler_output( - {req2.request_id: 1}, - new_reqs={req2.request_id: block_ids2}, - ) - meta2 = sched.build_connector_meta(sched_out2) - assert meta2.load_event >= 0 - assert req2.request_id in sched._reqs_to_load - - # Free request block refs (simulates preemption) - gpu_pool.free_blocks(gpu_pool.blocks[bid] for bid in block_ids[0]) - gpu_pool.free_blocks(gpu_pool.blocks[bid] for bid in block_ids2[0]) - - # Reset should keep load touch refs until the worker reports completion. - assert sched.reset() is False - assert len(sched._reqs_to_load) == 0 - assert len(sched._abandoned_reqs_to_load) == 1 - assert len(sched._load_event_to_reqs) == 1 - - num_used = gpu_pool.num_gpu_blocks - gpu_pool.get_num_free_blocks() - assert num_used > 1 - - simulate_load_completion(sched, {req2.request_id}) - assert len(sched._abandoned_reqs_to_load) == 0 - assert len(sched._load_event_to_reqs) == 0 - assert sched.reset() is True - - # All GPU blocks free - num_used = gpu_pool.num_gpu_blocks - gpu_pool.get_num_free_blocks() - assert num_used == 1, f"Expected only null block in use, got {num_used}" - - -def _make_cp_vllm_config( - dcp_world_size: int = 1, - pcp_world_size: int = 1, -) -> VllmConfig: - """VllmConfig with context-parallel sizes set for scheduler-only tests.""" - cfg = _make_vllm_config() - - cfg.parallel_config.decode_context_parallel_size = dcp_world_size - cfg.parallel_config.prefill_context_parallel_size = pcp_world_size - return cfg - - -def _make_cp_scheduler( - *, - dcp_world_size: int = 1, - pcp_world_size: int = 1, - num_cpu_blocks: int = 8, - num_gpu_blocks: int = 16, - lazy: bool = False, -) -> SchedulerFixture: - """Build a SimpleCPUOffloadScheduler with DCP-scaled block size.""" - virtual_block_size = BLOCK_SIZE * dcp_world_size - - kv_cache_config = _make_kv_cache_config(num_gpu_blocks) - vllm_config = _make_cp_vllm_config(dcp_world_size, pcp_world_size) - cpu_capacity_bytes = _BYTES_PER_BLOCK * num_cpu_blocks - - sched = SimpleCPUOffloadScheduler( - vllm_config=vllm_config, - kv_cache_config=kv_cache_config, - cpu_capacity_bytes=cpu_capacity_bytes, - scheduler_block_size=virtual_block_size, - hash_block_size=virtual_block_size, - lazy_offload=lazy, - ) - - gpu_block_pool = BlockPool( - num_gpu_blocks=num_gpu_blocks, - enable_caching=True, - hash_block_size=virtual_block_size, - ) - sched.bind_gpu_block_pool(gpu_block_pool) - - return SchedulerFixture( - scheduler=sched, - gpu_block_pool=gpu_block_pool, - vllm_config=vllm_config, - kv_cache_config=kv_cache_config, - ) - - -def _make_cp_request( - num_blocks: int, - virtual_block_size: int, - request_id: str | None = None, -) -> Request: - """Create a request whose block hashes are computed at the virtual - (CP-scaled) block size, matching what the real scheduler does. - """ - global _req_counter - _req_counter += 1 - if request_id is None: - request_id = f"req-cp-{_req_counter}" - - num_tokens = num_blocks * virtual_block_size + 1 - start = _req_counter * 10000 - prompt_token_ids = list(range(start, start + num_tokens)) - sampling_params = SamplingParams(max_tokens=1) - - return Request( - request_id=request_id, - prompt_token_ids=prompt_token_ids, - sampling_params=sampling_params, - pooling_params=None, - mm_features=None, - block_hasher=get_request_block_hasher(virtual_block_size, sha256), - ) - - -def _allocate_cp_gpu_blocks( - gpu_block_pool: BlockPool, - request: Request, - num_blocks: int, - virtual_block_size: int, - group_id: int = 0, -) -> list: - """Allocate GPU blocks and cache them using the CP-scaled block size.""" - blocks = gpu_block_pool.get_new_blocks(num_blocks) - num_full = min(num_blocks, len(request.block_hashes)) - if num_full > 0: - gpu_block_pool.cache_full_blocks( - request=request, - blocks=blocks, - num_cached_blocks=0, - num_full_blocks=num_full, - block_size=virtual_block_size, - kv_cache_group_id=group_id, - ) - return blocks - - -# --------------------------------------------------------------------------- -# Test 15: CP block size scaling is correct -# --------------------------------------------------------------------------- -@pytest.mark.parametrize( - "dcp_world_size, pcp_world_size", - [ - (2, 1), # DCP only - (1, 2), # PCP only - (2, 2), # DCP + PCP - ], -) -def test_cp_block_size_scaling(dcp_world_size: int, pcp_world_size: int) -> None: - """Verify block size scaling follows DCP ownership.""" - fix = _make_cp_scheduler( - dcp_world_size=dcp_world_size, pcp_world_size=pcp_world_size - ) - sched = fix.scheduler - - assert sched.cp_world_size == dcp_world_size - assert sched.block_size == BLOCK_SIZE * dcp_world_size - - -# --------------------------------------------------------------------------- -# Test 16: CP eager store-and-load roundtrip -# --------------------------------------------------------------------------- -@pytest.mark.parametrize( - "dcp_world_size, pcp_world_size", - [ - (2, 1), - (1, 2), - ], -) -def test_cp_eager_store_and_load_roundtrip( - dcp_world_size: int, pcp_world_size: int -) -> None: - """With CP enabled, store blocks to CPU and reload them for a new request - with matching tokens. Verifies that hash matching and transfer-pair - construction work with the virtual block size.""" - fix = _make_cp_scheduler( - dcp_world_size=dcp_world_size, - pcp_world_size=pcp_world_size, - num_cpu_blocks=8, - num_gpu_blocks=16, - lazy=False, - ) - sched = fix.scheduler - vbs = BLOCK_SIZE * dcp_world_size - - num_blocks = 2 - req = _make_cp_request(num_blocks, vbs) - - # Allocate GPU blocks and register hashes - gpu_blocks = _allocate_cp_gpu_blocks(fix.gpu_block_pool, req, num_blocks, vbs) - kv_blocks = KVCacheBlocks(blocks=(gpu_blocks,)) - req.num_computed_tokens = num_blocks * vbs - sched.update_state_after_alloc(req, kv_blocks, num_external_tokens=0) - - block_ids = kv_blocks.get_block_ids() - sched_out = make_scheduler_output( - {req.request_id: num_blocks * vbs}, - new_reqs={req.request_id: block_ids}, - ) - - meta = sched.build_connector_meta(sched_out) - assert meta.store_event >= 0, "Expected a store event" - assert len(meta.store_gpu_blocks) == num_blocks - assert len(meta.store_cpu_blocks) == num_blocks - simulate_store_completion(sched, meta.store_event) - - # New request with same tokens — should get a full CPU cache hit. - req2 = Request( - request_id="req-cp-load", - prompt_token_ids=req.prompt_token_ids, - sampling_params=req.sampling_params, - pooling_params=None, - mm_features=None, - block_hasher=req._block_hasher, - ) - - hit_tokens, is_async = sched.get_num_new_matched_tokens(req2, num_computed_tokens=0) - assert hit_tokens == num_blocks * vbs - assert is_async is True - - # Allocate fresh GPU blocks for the load. - gpu_blocks2 = fix.gpu_block_pool.get_new_blocks(num_blocks) - kv_blocks2 = KVCacheBlocks(blocks=(gpu_blocks2,)) - sched.update_state_after_alloc(req2, kv_blocks2, num_external_tokens=hit_tokens) - - sched_out2 = make_scheduler_output( - {req2.request_id: 1}, - new_reqs={req2.request_id: kv_blocks2.get_block_ids()}, - ) - meta2 = sched.build_connector_meta(sched_out2) - assert meta2.load_event >= 0, "Expected a load event" - assert len(meta2.load_gpu_blocks) == num_blocks - assert len(meta2.load_cpu_blocks) == num_blocks - - -# --------------------------------------------------------------------------- -# Test 18: CP store and load use effective block size -# --------------------------------------------------------------------------- -@pytest.mark.parametrize( - "dcp_world_size, pcp_world_size", - [ - (2, 1), - (1, 2), - (2, 2), - ], -) -def test_cp_effective_block_size_store_and_load( - dcp_world_size: int, pcp_world_size: int -) -> None: - """Verify store/load physical block counts scale with DCP, not PCP.""" - fix = _make_cp_scheduler( - dcp_world_size=dcp_world_size, pcp_world_size=pcp_world_size - ) - sched = fix.scheduler - gpu_pool = fix.gpu_block_pool - vbs = BLOCK_SIZE * dcp_world_size - expected_blocks = 1 - - # Store one DCP-scaled logical block worth of tokens. - req = _make_cp_request(num_blocks=2, virtual_block_size=vbs) - gpu_blocks = _allocate_cp_gpu_blocks(gpu_pool, req, 2, vbs) - kv = KVCacheBlocks(blocks=(gpu_blocks,)) - req.num_computed_tokens = vbs - sched.update_state_after_alloc(req, kv, num_external_tokens=0) - m1 = sched.build_connector_meta( - make_scheduler_output( - {req.request_id: vbs}, - new_reqs={req.request_id: kv.get_block_ids()}, - ) - ) - assert len(m1.store_gpu_blocks) == expected_blocks - assert len(m1.store_cpu_blocks) == expected_blocks - simulate_store_completion(sched, m1.store_event) - - # Load one DCP-scaled logical block from a two-block CPU hit. - req2 = _make_cp_request(num_blocks=2, virtual_block_size=vbs) - kv2 = KVCacheBlocks(blocks=(_allocate_cp_gpu_blocks(gpu_pool, req2, 2, vbs),)) - req2.num_computed_tokens = 2 * vbs - sched.update_state_after_alloc(req2, kv2, num_external_tokens=0) - m2 = sched.build_connector_meta( - make_scheduler_output( - {req2.request_id: 2 * vbs}, - new_reqs={req2.request_id: kv2.get_block_ids()}, - ) - ) - simulate_store_completion(sched, m2.store_event) - - req3 = Request( - request_id="req-cp-partial-load", - prompt_token_ids=req2.prompt_token_ids, - sampling_params=req2.sampling_params, - pooling_params=None, - mm_features=None, - block_hasher=req2._block_hasher, - ) - hit, _ = sched.get_num_new_matched_tokens(req3, num_computed_tokens=0) - assert hit == 2 * vbs - - kv3 = KVCacheBlocks(blocks=(gpu_pool.get_new_blocks(expected_blocks),)) - sched.update_state_after_alloc(req3, kv3, num_external_tokens=vbs) - m3 = sched.build_connector_meta( - make_scheduler_output( - {req3.request_id: vbs}, - new_reqs={req3.request_id: kv3.get_block_ids()}, - ) - ) - assert m3.load_event >= 0 - assert len(m3.load_gpu_blocks) == expected_blocks - assert len(m3.load_cpu_blocks) == expected_blocks - assert m3.load_gpu_blocks == [kv3.get_block_ids()[0][0]] - - -# --------------------------------------------------------------------------- -# Test 17: CP lazy target blocks are scaled correctly -# --------------------------------------------------------------------------- -@pytest.mark.parametrize("cp_world_size", [1, 2, 4]) -def test_cp_lazy_target_blocks_scaling(cp_world_size: int) -> None: - """_estimate_lazy_target_blocks returns fewer blocks when cp_world_size > 1 - because each virtual block covers more tokens.""" - kv_cache_config = _make_kv_cache_config(num_blocks=16) - max_batched = 64 - - target_base = SimpleCPUOffloadScheduler._estimate_lazy_target_blocks( - kv_cache_config, max_batched, cp_world_size=1 - ) - target_cp = SimpleCPUOffloadScheduler._estimate_lazy_target_blocks( - kv_cache_config, max_batched, cp_world_size=cp_world_size - ) - - if cp_world_size == 1: - assert target_cp == target_base - else: - assert target_cp < target_base, ( - f"cp_world_size={cp_world_size}: target_cp={target_cp} should be " - f"less than target_base={target_base}" - ) - - -def _make_hybrid_attention_mamba_scheduler( - *, - num_cpu_blocks: int = 8, - num_gpu_blocks: int = 16, - attention_block_size: int = BLOCK_SIZE, - block_size: int = 4 * BLOCK_SIZE, - scheduler_block_size: int | None = None, - hash_block_size: int | None = None, - dcp_world_size: int = 4, - lazy: bool = False, - mamba_cache_mode: MambaCacheMode = "align", - enable_kv_cache_events: bool = False, -) -> SchedulerFixture: - """Build a scheduler for one attention group plus one Mamba group.""" - attention_spec = FullAttentionSpec( - block_size=attention_block_size, - num_kv_heads=NUM_KV_HEADS, - head_size=HEAD_SIZE, - dtype=DTYPE, - ) - mamba_spec = MambaSpec( - block_size=block_size, - shapes=((1, 1),), - dtypes=(torch.float32,), - mamba_cache_mode=mamba_cache_mode, - ) - return _make_two_group_scheduler( - attention_spec, - ("mamba", mamba_spec), - num_cpu_blocks=num_cpu_blocks, - num_gpu_blocks=num_gpu_blocks, - scheduler_block_size=scheduler_block_size or block_size, - hash_block_size=hash_block_size or block_size, - dcp_world_size=dcp_world_size, - lazy=lazy, - mamba_cache_mode=mamba_cache_mode, - enable_kv_cache_events=enable_kv_cache_events, - ) - - -def _make_two_group_scheduler( - attention_spec: FullAttentionSpec, - second_group: tuple[str, KVCacheSpec], - *, - num_cpu_blocks: int, - num_gpu_blocks: int, - scheduler_block_size: int, - hash_block_size: int, - dcp_world_size: int, - lazy: bool = False, - mamba_cache_mode: MambaCacheMode = "align", - enable_kv_cache_events: bool = False, -) -> SchedulerFixture: - """Build a scheduler for one attention group plus one named second group.""" - second_name, second_spec = second_group - groups = [ - KVCacheGroupSpec(["attention"], attention_spec), - KVCacheGroupSpec([second_name], second_spec), - ] - tensors = [ - KVCacheTensor( - size=spec.page_size_bytes * num_gpu_blocks, - layers=group.layer_names, - layer_stride=spec.page_size_bytes * num_gpu_blocks, - block_stride=spec.page_size_bytes, - ) - for group, spec in zip(groups, (attention_spec, second_spec)) - ] - kv_cache_config = KVCacheConfig( - num_blocks=num_gpu_blocks, - kv_cache_tensors=tensors, - kv_cache_groups=groups, - ) - vllm_config = _make_cp_vllm_config(dcp_world_size=dcp_world_size) - vllm_config.cache_config.prefix_cache_retention_interval = 0 - vllm_config.cache_config.mamba_cache_mode = mamba_cache_mode - if enable_kv_cache_events: - vllm_config.kv_events_config = KVEventsConfig(enable_kv_cache_events=True) - # _derive_cpu_config() scales the requested capacity against - # kv_cache_tensors[0].size, so express it as a multiple of that tensor's - # per-block size to get exactly num_cpu_blocks. - cpu_capacity_bytes = tensors[0].size // num_gpu_blocks * num_cpu_blocks - sched = SimpleCPUOffloadScheduler( - vllm_config=vllm_config, - kv_cache_config=kv_cache_config, - cpu_capacity_bytes=cpu_capacity_bytes, - scheduler_block_size=scheduler_block_size, - hash_block_size=hash_block_size, - lazy_offload=lazy, - ) - gpu_block_pool = BlockPool( - num_gpu_blocks=num_gpu_blocks, - enable_caching=True, - hash_block_size=hash_block_size, - ) - sched.bind_gpu_block_pool(gpu_block_pool) - return SchedulerFixture( - scheduler=sched, - gpu_block_pool=gpu_block_pool, - vllm_config=vllm_config, - kv_cache_config=kv_cache_config, - ) - - -def test_hybrid_store_uses_resolved_group_block_sizes() -> None: - """Replicated groups must not be scaled by the DCP world size. - - The spec's ``dcp_sharded`` flag controls its effective block size. - Reconstructing a group's geometry as - ``spec.block_size * cp_world_size`` over-scales the replicated groups, so - the eager store scan believes far fewer of their blocks are ready and - silently offloads only a fraction of them. - """ - attention_block_size = BLOCK_SIZE - replicated_block_size = 4 * BLOCK_SIZE - dcp_world_size = 2 - fix = _make_two_group_scheduler( - FullAttentionSpec( - block_size=attention_block_size, - num_kv_heads=NUM_KV_HEADS, - head_size=HEAD_SIZE, - dtype=DTYPE, - ), - ( - "replicated_attention", - FullAttentionSpec( - block_size=replicated_block_size, - num_kv_heads=NUM_KV_HEADS, - head_size=HEAD_SIZE, - dtype=DTYPE, - dcp_sharded=False, - ), - ), - num_cpu_blocks=32, - num_gpu_blocks=32, - scheduler_block_size=replicated_block_size, - # Every prefix-cacheable group's resolved block size must be a multiple - # of the hash block, so it cannot exceed the smallest of them. - hash_block_size=BLOCK_SIZE, - dcp_world_size=dcp_world_size, - ) - sched = fix.scheduler - gpu_pool = fix.gpu_block_pool - attention_size, replicated_size = sched.group_block_sizes - assert attention_size == attention_block_size * dcp_world_size - assert replicated_size == replicated_block_size - - confirmed = 2 * sched.block_size - req = _make_cp_request(num_blocks=8, virtual_block_size=BLOCK_SIZE) - attention_blocks = _allocate_cp_gpu_blocks( - gpu_pool, req, confirmed // attention_size, attention_size, group_id=0 - ) - replicated_blocks = _allocate_cp_gpu_blocks( - gpu_pool, req, confirmed // replicated_size, replicated_size, group_id=1 - ) - kv_blocks = KVCacheBlocks(blocks=(attention_blocks, replicated_blocks)) - sched.update_state_after_alloc(req, kv_blocks, num_external_tokens=0) - sched.build_connector_meta( - make_scheduler_output( - {req.request_id: confirmed}, - new_reqs={req.request_id: kv_blocks.get_block_ids()}, - ) - ) - req.num_computed_tokens = confirmed - meta = sched.build_connector_meta( - make_scheduler_output( - {req.request_id: 1}, - cached_req_new_blocks={req.request_id: None}, - ) - ) - - # Both groups are positionally stable here, so every confirmed block of - # both must be offloaded. Over-scaling the replicated group halves its - # count. - assert sched._reqs_to_store[req.request_id].num_stored_blocks == [ - confirmed // attention_size, - confirmed // replicated_size, - ] - assert set(meta.store_gpu_blocks) == { - *(b.block_id for b in attention_blocks), - *(b.block_id for b in replicated_blocks), - } - - -def test_dcp_mixed_cache_loads_fine_grained_external_hit() -> None: - """Map a fine-grained external suffix after a nonzero local prefix.""" - hash_block_size = BLOCK_SIZE - mamba_block_size = 4 * BLOCK_SIZE - local_tokens = 4 * BLOCK_SIZE - # Span two Mamba blocks and end partially in the second one. This exercises - # both cdiv(external_tokens, mamba_block_size) > 1 and the partial tail. - external_tokens = 6 * BLOCK_SIZE - total_cached_tokens = local_tokens + external_tokens - fix = _make_hybrid_attention_mamba_scheduler( - num_cpu_blocks=32, - num_gpu_blocks=32, - block_size=mamba_block_size, - hash_block_size=hash_block_size, - dcp_world_size=2, - ) - sched = fix.scheduler - gpu_pool = fix.gpu_block_pool - - assert external_tokens % mamba_block_size != 0 - assert sched.cpu_coordinator.enable_partial_hash_hits - - producer = _make_cp_request(num_blocks=10, virtual_block_size=hash_block_size) - attention_blocks = _allocate_cp_gpu_blocks( - gpu_pool, - producer, - num_blocks=5, - virtual_block_size=2 * BLOCK_SIZE, - group_id=0, - ) - mamba_blocks = gpu_pool.get_new_blocks(3) - gpu_pool.cache_full_blocks( - request=producer, - blocks=mamba_blocks, - num_cached_blocks=0, - num_full_blocks=2, - block_size=mamba_block_size, - kv_cache_group_id=1, - ) - gpu_pool.cache_partial_block( - request=producer, - block=mamba_blocks[2], - num_tokens=total_cached_tokens, - kv_cache_group_id=1, - block_size=mamba_block_size, - ) - producer_blocks = KVCacheBlocks(blocks=(attention_blocks, mamba_blocks)) - sched.update_state_after_alloc(producer, producer_blocks, num_external_tokens=0) - - # Positional scanning stores append-only FA blocks. Mamba align blocks use - # exact handoffs for every retained boundary, including the partial tail. - store_output = make_scheduler_output( - {producer.request_id: total_cached_tokens}, - new_reqs={producer.request_id: producer_blocks.get_block_ids()}, - ) - store_output.kv_connector_block_state = KVConnectorBlockState( - req_ids=set(), - resolve_block_ids=lambda _: (), - boundary_state_offloads={ - producer.request_id: [ - (1, mamba_blocks[0].block_id, mamba_block_size), - (1, mamba_blocks[1].block_id, 2 * mamba_block_size), - (1, mamba_blocks[2].block_id, total_cached_tokens), - ] - }, - ) - store_meta = sched.build_connector_meta(store_output) - # The exact handoffs land in the step they arrive; the append-only FA scan - # picks its blocks up once the scheduler has committed the tokens. - assert set(store_meta.store_gpu_blocks) == { - block.block_id for block in mamba_blocks - } - simulate_store_completion(sched, store_meta.store_event) - producer.num_computed_tokens = total_cached_tokens - fa_meta = sched.build_connector_meta( - make_scheduler_output( - {producer.request_id: 1}, - cached_req_new_blocks={producer.request_id: None}, - ) - ) - assert set(fa_meta.store_gpu_blocks) == { - block.block_id for block in attention_blocks - } - simulate_store_completion(sched, fa_meta.store_event) - - consumer = Request( - request_id="req-dcp-mixed-fine-load", - prompt_token_ids=producer.prompt_token_ids, - sampling_params=producer.sampling_params, - pooling_params=None, - mm_features=None, - block_hasher=producer._block_hasher, - ) - hit_tokens, is_async = sched.get_num_new_matched_tokens( - consumer, num_computed_tokens=local_tokens - ) - assert hit_tokens == external_tokens - assert is_async is True - pending_blocks, pending_tokens, _ = sched._pending_cpu_hits[consumer.request_id] - assert pending_tokens == external_tokens - assert len(pending_blocks[1]) == 2 - assert pending_blocks[1][0].is_null - assert not pending_blocks[1][1].is_null - - local_attention = attention_blocks[:2] - local_mamba = mamba_blocks[:1] - load_blocks = KVCacheBlocks( - blocks=( - local_attention + gpu_pool.get_new_blocks(3), - local_mamba + gpu_pool.get_new_blocks(2), - ) - ) - sched.update_state_after_alloc( - consumer, - load_blocks, - num_external_tokens=hit_tokens, - ) - load_meta = sched.build_connector_meta( - make_scheduler_output( - {consumer.request_id: 1}, - new_reqs={consumer.request_id: load_blocks.get_block_ids()}, - ) - ) - assert load_meta.load_gpu_blocks == [ - *(block.block_id for block in load_blocks.blocks[0][2:5]), - load_blocks.blocks[1][2].block_id, - ] - assert len(load_meta.load_cpu_blocks) == 4 - - -def test_finished_eager_store_caches_fa_partial_tail_for_hybrid_hit() -> None: - """A finished non-LCM prompt needs the FA partial tail in CPU too.""" - hash_block_size = BLOCK_SIZE - mamba_block_size = BLOCK_SIZE - attention_block_size = 4 * BLOCK_SIZE - dcp_world_size = 2 - scheduler_block_size = attention_block_size * dcp_world_size - total_cached_tokens = 5 * hash_block_size - fix = _make_hybrid_attention_mamba_scheduler( - num_cpu_blocks=32, - num_gpu_blocks=32, - attention_block_size=attention_block_size, - block_size=mamba_block_size, - scheduler_block_size=scheduler_block_size, - hash_block_size=hash_block_size, - dcp_world_size=dcp_world_size, - ) - sched = fix.scheduler - gpu_pool = fix.gpu_block_pool - assert sched.fa_block_size == scheduler_block_size - assert total_cached_tokens % sched.fa_block_size != 0 - - producer = _make_cp_request(num_blocks=5, virtual_block_size=hash_block_size) - attention_blocks = gpu_pool.get_new_blocks(1) - gpu_pool.cache_partial_block( - request=producer, - block=attention_blocks[0], - num_tokens=total_cached_tokens, - kv_cache_group_id=0, - block_size=scheduler_block_size, - ) - mamba_blocks = gpu_pool.get_new_blocks(5) - gpu_pool.cache_full_blocks( - request=producer, - blocks=mamba_blocks, - num_cached_blocks=0, - num_full_blocks=5, - kv_cache_group_id=1, - block_size=mamba_block_size, - ) - producer_blocks = KVCacheBlocks(blocks=(attention_blocks, mamba_blocks)) - sched.update_state_after_alloc(producer, producer_blocks, num_external_tokens=0) - - store_output = make_scheduler_output( - {producer.request_id: total_cached_tokens}, - new_reqs={producer.request_id: producer_blocks.get_block_ids()}, - ) - store_output.kv_connector_block_state = KVConnectorBlockState( - req_ids=set(), - resolve_block_ids=lambda _: (), - boundary_state_offloads={ - producer.request_id: [ - (1, mamba_blocks[4].block_id, total_cached_tokens), - ] - }, - ) - store_meta = sched.build_connector_meta(store_output) - simulate_store_completion(sched, store_meta.store_event) - - producer.num_computed_tokens = total_cached_tokens - sched.request_finished_all_groups(producer, producer_blocks.get_block_ids()) - finish_meta = sched.build_connector_meta(make_scheduler_output({})) - assert attention_blocks[0].block_id in finish_meta.store_gpu_blocks - simulate_store_completion(sched, finish_meta.store_event) - - consumer = Request( - request_id="req-finished-partial-tail-load", - prompt_token_ids=producer.prompt_token_ids, - sampling_params=producer.sampling_params, - pooling_params=None, - mm_features=None, - block_hasher=producer._block_hasher, - ) - hit_tokens, is_async = sched.get_num_new_matched_tokens( - consumer, num_computed_tokens=0 - ) - assert hit_tokens == total_cached_tokens - assert is_async is True - - -def test_finished_eager_store_does_not_duplicate_completed_partial_tail() -> None: - """Decode can fill the boundary block; the scan then already covers it.""" - hash_block_size = BLOCK_SIZE - mamba_block_size = BLOCK_SIZE - attention_block_size = 4 * BLOCK_SIZE - dcp_world_size = 2 - scheduler_block_size = attention_block_size * dcp_world_size - prompt_tokens = 5 * hash_block_size - fix = _make_hybrid_attention_mamba_scheduler( - num_cpu_blocks=32, - num_gpu_blocks=32, - attention_block_size=attention_block_size, - block_size=mamba_block_size, - scheduler_block_size=scheduler_block_size, - hash_block_size=hash_block_size, - dcp_world_size=dcp_world_size, - ) - sched = fix.scheduler - gpu_pool = fix.gpu_block_pool - assert prompt_tokens % sched.fa_block_size != 0 - - producer = _make_cp_request(num_blocks=5, virtual_block_size=hash_block_size) - attention_blocks = gpu_pool.get_new_blocks(1) - gpu_pool.cache_partial_block( - request=producer, - block=attention_blocks[0], - num_tokens=prompt_tokens, - kv_cache_group_id=0, - block_size=scheduler_block_size, - ) - mamba_blocks = gpu_pool.get_new_blocks(5) - gpu_pool.cache_full_blocks( - request=producer, - blocks=mamba_blocks, - num_cached_blocks=0, - num_full_blocks=5, - kv_cache_group_id=1, - block_size=mamba_block_size, - ) - producer_blocks = KVCacheBlocks(blocks=(attention_blocks, mamba_blocks)) - sched.update_state_after_alloc(producer, producer_blocks, num_external_tokens=0) - - # Decode carries the request past the physical attention block boundary, so - # the positional scan now considers the same block the partial tail names. - producer.num_computed_tokens = sched.fa_block_size - sched.request_finished_all_groups(producer, producer_blocks.get_block_ids()) - finish_meta = sched.build_connector_meta(make_scheduler_output({})) - - stored = finish_meta.store_gpu_blocks - assert attention_blocks[0].block_id in stored - assert len(stored) == len(set(stored)) - assert len(finish_meta.store_cpu_blocks) == len(stored) - - -def test_external_lookup_rejects_misaligned_local_prefix(caplog_vllm) -> None: - scheduler_block_size = 4 * BLOCK_SIZE - fix = _make_hybrid_attention_mamba_scheduler(block_size=scheduler_block_size) - - request = _make_cp_request(num_blocks=2, virtual_block_size=scheduler_block_size) - # The ``vllm`` logger sets propagate=False, so plain ``caplog`` can observe - # nothing; ``caplog_vllm`` re-enables propagation for the duration. - with caplog_vllm.at_level(logging.WARNING): - hit_tokens, is_async = fix.scheduler.get_num_new_matched_tokens( - request, num_computed_tokens=BLOCK_SIZE - ) - - assert (hit_tokens, is_async) == (0, False) - assert "requires scheduler-block-aligned local tokens" in caplog_vllm.text - - -def test_fine_grained_external_hits_require_eager_offload() -> None: - hash_block_size = BLOCK_SIZE - scheduler_block_size = 4 * BLOCK_SIZE - - eager = _make_hybrid_attention_mamba_scheduler( - block_size=scheduler_block_size, - hash_block_size=hash_block_size, - ) - lazy = _make_hybrid_attention_mamba_scheduler( - block_size=scheduler_block_size, - hash_block_size=hash_block_size, - lazy=True, - ) - - assert eager.scheduler.cpu_coordinator.enable_partial_hash_hits - assert not lazy.scheduler.cpu_coordinator.enable_partial_hash_hits - - -def test_eager_store_does_not_scan_mamba_blocks_positionally() -> None: - """Mamba stores must use exact handoffs, not historical block positions.""" - block_size = 4 * BLOCK_SIZE - fix = _make_hybrid_attention_mamba_scheduler( - num_cpu_blocks=16, num_gpu_blocks=24, block_size=block_size - ) - sched = fix.scheduler - gpu_pool = fix.gpu_block_pool - attention_manager, mamba_manager = sched.cpu_coordinator.single_type_managers - assert attention_manager.has_positionally_stable_blocks - assert not mamba_manager.has_positionally_stable_blocks - - req = _make_cp_request(num_blocks=2, virtual_block_size=block_size) - attn_blocks = _allocate_cp_gpu_blocks( - gpu_pool, req, 2, virtual_block_size=block_size, group_id=0 - ) - gpu_blocks = gpu_pool.get_new_blocks(2) - gpu_blocks[1].set_block_hash(make_block_hash_with_group_id(req.block_hashes[1], 1)) - kv_blocks = KVCacheBlocks(blocks=(attn_blocks, gpu_blocks)) - req.num_computed_tokens = 0 - sched.update_state_after_alloc(req, kv_blocks, num_external_tokens=0) - - output = make_scheduler_output( - {req.request_id: 2 * block_size}, - new_reqs={req.request_id: kv_blocks.get_block_ids()}, - ) - output.kv_connector_block_state = KVConnectorBlockState( - req_ids=set(), - resolve_block_ids=lambda _: (), - boundary_state_offloads={ - req.request_id: [(1, gpu_blocks[1].block_id, 2 * block_size)] - }, - ) - meta1 = sched.build_connector_meta(output) - assert meta1.store_event >= 0 - # Only the exact handoff lands this step. build_connector_meta runs before - # the scheduler commits this step's tokens, so the positional scan has - # nothing confirmed yet. - assert set(meta1.store_gpu_blocks) == {gpu_blocks[1].block_id} - assert sched._reqs_to_store[req.request_id].num_stored_blocks == [0, 0] - simulate_store_completion(sched, meta1.store_event) - - # Next step the attention blocks are confirmed and scanned positionally. A - # later hash update still must not make the mamba align snapshot a valid - # source; the explicit boundary-handoff tests cover the supported path. - gpu_blocks[0].set_block_hash(make_block_hash_with_group_id(req.block_hashes[0], 1)) - req.num_computed_tokens = 2 * block_size - meta2 = sched.build_connector_meta( - make_scheduler_output( - {req.request_id: 1}, - cached_req_new_blocks={req.request_id: None}, - ) - ) - assert set(meta2.store_gpu_blocks) == {block.block_id for block in attn_blocks} - assert sched._reqs_to_store[req.request_id].num_stored_blocks == [2, 0] - simulate_store_completion(sched, meta2.store_event) - - # Nothing further: mamba align is never picked up positionally. - meta3 = sched.build_connector_meta( - make_scheduler_output( - {req.request_id: 1}, - cached_req_new_blocks={req.request_id: None}, - ) - ) - assert meta3.store_event == -1 - assert meta3.store_gpu_blocks == [] - assert sched._reqs_to_store[req.request_id].num_stored_blocks == [2, 0] - - -@pytest.mark.parametrize("gone_via", ["preempted", "finished", "unregistered"]) -def test_boundary_handoff_dropped_for_departing_request(gone_via: str) -> None: - """Handoffs for a request leaving this step must not be read. - - The scheduler drains boundary handoffs without filtering by request - liveness, so a preempted or finished request can still offer block IDs. - Those blocks are being released and may already back another request, so - copying them would publish unrelated KV under a valid hash. - """ - block_size = 4 * BLOCK_SIZE - fix = _make_hybrid_attention_mamba_scheduler( - num_cpu_blocks=16, num_gpu_blocks=24, block_size=block_size - ) - sched = fix.scheduler - gpu_pool = fix.gpu_block_pool - - req = _make_cp_request(num_blocks=2, virtual_block_size=block_size) - attn_blocks = _allocate_cp_gpu_blocks( - gpu_pool, req, 2, virtual_block_size=block_size, group_id=0 - ) - mamba_blocks = gpu_pool.get_new_blocks(2) - mamba_blocks[1].set_block_hash( - make_block_hash_with_group_id(req.block_hashes[1], 1) - ) - kv_blocks = KVCacheBlocks(blocks=(attn_blocks, mamba_blocks)) - sched.update_state_after_alloc(req, kv_blocks, num_external_tokens=0) - - output = make_scheduler_output( - {req.request_id: 2 * block_size}, - new_reqs={req.request_id: kv_blocks.get_block_ids()}, - ) - output.kv_connector_block_state = KVConnectorBlockState( - req_ids=set(), - resolve_block_ids=lambda _: (), - boundary_state_offloads={ - req.request_id: [(1, mamba_blocks[1].block_id, 2 * block_size)] - }, - ) - if gone_via == "preempted": - output.preempted_req_ids = {req.request_id} - elif gone_via == "finished": - output.finished_req_ids = {req.request_id} - else: - sched._reqs_to_store.pop(req.request_id) - - meta = sched.build_connector_meta(output) - - # The exact-handoff block is never a store source for a departing request. - assert mamba_blocks[1].block_id not in meta.store_gpu_blocks - stats = sched.get_boundary_store_stats() - assert stats.dropped_request_gone == 1 - assert stats.stored == 0 - - -def test_boundary_handoff_keeps_block_meta_index_parallel() -> None: - """Boundary stores must extend block_meta alongside the block ids. - - ``_process_store_completion`` indexes ``block_meta`` by position, so a - boundary handoff that appends only block ids shortens the list, trips its - length assertion and shifts every following entry's metadata. - """ - block_size = 4 * BLOCK_SIZE - fix = _make_hybrid_attention_mamba_scheduler( - num_cpu_blocks=16, - num_gpu_blocks=24, - block_size=block_size, - enable_kv_cache_events=True, - ) - sched = fix.scheduler - gpu_pool = fix.gpu_block_pool - assert sched.enable_kv_cache_events - - req = _make_cp_request(num_blocks=2, virtual_block_size=block_size) - attn_blocks = _allocate_cp_gpu_blocks( - gpu_pool, req, 2, virtual_block_size=block_size, group_id=0 - ) - mamba_blocks = gpu_pool.get_new_blocks(2) - mamba_blocks[1].set_block_hash( - make_block_hash_with_group_id(req.block_hashes[1], 1) - ) - kv_blocks = KVCacheBlocks(blocks=(attn_blocks, mamba_blocks)) - sched.update_state_after_alloc(req, kv_blocks, num_external_tokens=0) - # Make the FA blocks positionally eligible in the same event as the - # explicit Mamba boundary handoff. - req.num_computed_tokens = 2 * block_size - - output = make_scheduler_output( - {req.request_id: 2 * block_size}, - new_reqs={req.request_id: kv_blocks.get_block_ids()}, - ) - output.kv_connector_block_state = KVConnectorBlockState( - req_ids=set(), - resolve_block_ids=lambda _: (), - boundary_state_offloads={ - req.request_id: [(1, mamba_blocks[1].block_id, 2 * block_size)] - }, - ) - - gpu_ids, cpu_ids, req_ids, block_meta = sched.prepare_store_specs(output) - assert mamba_blocks[1].block_id in gpu_ids - assert set(gpu_ids) == { - *(block.block_id for block in attn_blocks), - mamba_blocks[1].block_id, - } - assert req_ids == [req.request_id] - assert block_meta is not None - assert len(block_meta) == len(gpu_ids) == len(cpu_ids) - - -def test_finished_eager_store_recovers_nulled_window_tail() -> None: - """A sliding-window group nulls pages that left the window before the - connector sees the finished request's block table, while the GPU keeps - them hashed in its free queue. The eager store must copy those pages, or - the CPU lookup (which needs the whole window) misses every time.""" - fix = make_scheduler(num_cpu_blocks=16, num_gpu_blocks=32, num_groups=2, lazy=False) - sched = fix.scheduler - gpu_pool = fix.gpu_block_pool - null_block = gpu_pool.null_block - - num_blocks = 4 # == sliding window of the test SWA group - req = make_request(num_blocks=num_blocks) - fa_blocks = _allocate_gpu_blocks(gpu_pool, req, num_blocks, group_id=0) - swa_blocks = _allocate_gpu_blocks(gpu_pool, req, num_blocks, group_id=1) - # remove_skipped_blocks() ran: the two oldest window pages are freed (still - # hashed) and their slots hold the null block. - gpu_pool.free_blocks(swa_blocks[:2]) - swa_table = [null_block, null_block, *swa_blocks[2:]] - kv_blocks = KVCacheBlocks(blocks=(fa_blocks, swa_table)) - req.num_computed_tokens = num_blocks * BLOCK_SIZE - sched.update_state_after_alloc(req, kv_blocks, num_external_tokens=0) - - block_ids = kv_blocks.get_block_ids() - sched.request_finished_all_groups(req, block_ids) - meta = sched.build_connector_meta(make_scheduler_output({})) - - assert null_block.block_id not in meta.store_gpu_blocks - assert set(meta.store_gpu_blocks) == {b.block_id for b in (*fa_blocks, *swa_blocks)} - assert meta.store_event >= 0 - simulate_store_completion(sched, meta.store_event) - - req2 = Request( - request_id="req-window-tail-load", - prompt_token_ids=req.prompt_token_ids, - sampling_params=req.sampling_params, - pooling_params=None, - mm_features=None, - block_hasher=req._block_hasher, - ) - hit_tokens, is_async = sched.get_num_new_matched_tokens(req2, num_computed_tokens=0) - assert hit_tokens == num_blocks * BLOCK_SIZE - assert is_async is True - - -QSA_RING_BLOCK_SIZE = 4 - - -def _make_qsa_hybrid_kv_cache_config(num_blocks: int) -> KVCacheConfig: - register_all_kvcache_specs(vllm_config=None) - full_spec = FullAttentionSpec( - block_size=BLOCK_SIZE, - num_kv_heads=NUM_KV_HEADS, - head_size=HEAD_SIZE, - dtype=DTYPE, - ) - ring_spec = CircularBufferSpec( - block_size=QSA_RING_BLOCK_SIZE, - num_kv_heads=NUM_KV_HEADS, - head_size=HEAD_SIZE, - head_size_v=0, - dtype=DTYPE, - ) - groups = [ - KVCacheGroupSpec(["layer_0"], full_spec), - KVCacheGroupSpec(["qsa_ring"], ring_spec), - ] - tensors = [ - KVCacheTensor( - size=group.kv_cache_spec.page_size_bytes * num_blocks, - layers=list(group.layer_names), - layer_stride=group.kv_cache_spec.page_size_bytes * num_blocks, - block_stride=group.kv_cache_spec.page_size_bytes, - ) - for group in groups - ] - return KVCacheConfig( - num_blocks=num_blocks, - kv_cache_tensors=tensors, - kv_cache_groups=groups, - ) - - -def _make_qsa_scheduler( - lazy: bool = False, -) -> tuple[SimpleCPUOffloadScheduler, BlockPool]: - kv_cache_config = _make_qsa_hybrid_kv_cache_config(num_blocks=16) - sched = SimpleCPUOffloadScheduler( - vllm_config=_make_vllm_config(), - kv_cache_config=kv_cache_config, - cpu_capacity_bytes=_BYTES_PER_BLOCK * 8 * 2, - scheduler_block_size=BLOCK_SIZE, - hash_block_size=BLOCK_SIZE, - lazy_offload=lazy, - ) - gpu_pool = BlockPool( - num_gpu_blocks=16, enable_caching=True, hash_block_size=BLOCK_SIZE - ) - sched.bind_gpu_block_pool(gpu_pool) - return sched, gpu_pool - - -def test_qsa_ring_group_is_never_stored_or_loaded() -> None: - sched, gpu_pool = _make_qsa_scheduler() - assert sched.prefix_cacheable_group_ids == (0,) - - num_blocks = 2 - req = make_request(num_blocks=num_blocks) - fa_blocks = _allocate_gpu_blocks(gpu_pool, req, num_blocks, group_id=0) - ring_block = gpu_pool.get_new_blocks(1)[0] - kv_blocks = KVCacheBlocks(blocks=(fa_blocks, [ring_block])) - req.num_computed_tokens = num_blocks * BLOCK_SIZE - sched.update_state_after_alloc(req, kv_blocks, num_external_tokens=0) - - sched_out = make_scheduler_output( - {req.request_id: num_blocks * BLOCK_SIZE}, - new_reqs={req.request_id: kv_blocks.get_block_ids()}, - ) - meta = sched.build_connector_meta(sched_out) - assert set(meta.store_gpu_blocks) == {block.block_id for block in fa_blocks} - assert ring_block.block_id not in meta.store_gpu_blocks - assert meta.store_event >= 0 - simulate_store_completion(sched, meta.store_event) - - req2 = Request( - request_id="req-qsa-load", - prompt_token_ids=req.prompt_token_ids, - sampling_params=req.sampling_params, - pooling_params=None, - mm_features=None, - block_hasher=req._block_hasher, - ) - hit_tokens, is_async = sched.get_num_new_matched_tokens(req2, num_computed_tokens=0) - assert hit_tokens == num_blocks * BLOCK_SIZE - assert is_async is True - - fa_blocks2 = gpu_pool.get_new_blocks(num_blocks) - ring_block2 = gpu_pool.get_new_blocks(1)[0] - kv_blocks2 = KVCacheBlocks(blocks=(fa_blocks2, [ring_block2])) - sched.update_state_after_alloc(req2, kv_blocks2, num_external_tokens=hit_tokens) - - meta2 = sched.build_connector_meta(make_scheduler_output({req2.request_id: 1})) - assert set(meta2.load_gpu_blocks) == {block.block_id for block in fa_blocks2} - assert ring_block2.block_id not in meta2.load_gpu_blocks - assert len(meta2.load_cpu_blocks) == num_blocks - - -def test_lazy_target_blocks_ignore_non_prefix_cacheable_groups() -> None: - with_ring = _make_qsa_hybrid_kv_cache_config(num_blocks=16) - without_ring = KVCacheConfig( - num_blocks=16, - kv_cache_tensors=with_ring.kv_cache_tensors[:1], - kv_cache_groups=with_ring.kv_cache_groups[:1], - ) - max_batched = 64 - target_with = SimpleCPUOffloadScheduler._estimate_lazy_target_blocks( - with_ring, max_batched - ) - target_without = SimpleCPUOffloadScheduler._estimate_lazy_target_blocks( - without_ring, max_batched - ) - assert target_with == target_without - - -@pytest.mark.parametrize("retention_interval", [None, 0]) -@pytest.mark.parametrize("chunk_size,prefill_lookahead", [(64, 1), (1024, 1), (1, 3)]) -@pytest.mark.parametrize( - "draft_sliding_window,draft_sharded", - [(True, False), (False, False), (False, True)], - ids=["swa", "replicated_full", "sharded_full"], -) -def test_eager_store_matches_gpu_cacheable_prefix( - retention_interval: int | None, - chunk_size: int, - prefill_lookahead: int, - draft_sliding_window: bool, - draft_sharded: bool, -) -> None: - """Preserve GPU hits for replicated SWA and DCP-sharded full-attention drafts.""" - block_size = 64 - scheduler_block_size = 8 * block_size - draft_block_size = scheduler_block_size if draft_sharded else block_size - prompt_len = 16 * scheduler_block_size + draft_block_size + prefill_lookahead - kv_cache_config = _make_kv_cache_config(256, num_groups=2) - for group_id, group in enumerate(kv_cache_config.kv_cache_groups): - group.kv_cache_spec = FullAttentionSpec( - block_size=block_size, - num_kv_heads=NUM_KV_HEADS, - head_size=HEAD_SIZE, - dtype=DTYPE, - dcp_sharded=group_id == 0 or draft_sharded, - ) - for tensor in kv_cache_config.kv_cache_tensors: - tensor.size *= block_size // BLOCK_SIZE - tensor.layer_stride *= block_size // BLOCK_SIZE - tensor.block_stride *= block_size // BLOCK_SIZE - if draft_sliding_window: - kv_cache_config.kv_cache_groups[1].kv_cache_spec = SlidingWindowSpec( - block_size=block_size, - num_kv_heads=NUM_KV_HEADS, - head_size=HEAD_SIZE, - dtype=DTYPE, - sliding_window=scheduler_block_size, - extra_retained_tokens=prefill_lookahead - 1, - dcp_sharded=False, - ) - kv_cache_config.kv_cache_groups[1].is_eagle_group = True - kv_cache_config.prefix_cache_retention_interval = retention_interval - vllm_config = _make_cp_vllm_config(dcp_world_size=8) - vllm_config.model_config.max_model_len = 16384 - speculative_config = Mock(spec=SpeculativeConfig) - speculative_config.num_speculative_tokens = 3 - speculative_config.use_eagle.return_value = True - speculative_config.use_eagle_block_drop.return_value = True - speculative_config.use_multi_module_mtp.return_value = prefill_lookahead > 1 - vllm_config.speculative_config = speculative_config - scheduler = SimpleCPUOffloadScheduler( - vllm_config=vllm_config, - kv_cache_config=kv_cache_config, - cpu_capacity_bytes=kv_cache_config.kv_cache_tensors[0].size, - scheduler_block_size=scheduler_block_size, - hash_block_size=block_size, - ) - gpu_manager = KVCacheManager( - kv_cache_config, - max_model_len=16384, - max_in_flight_tokens=chunk_size, - scheduler_block_size=scheduler_block_size, - hash_block_size=block_size, - dcp_world_size=8, - use_eagle=True, - num_prefill_lookahead=prefill_lookahead, - ) - scheduler.bind_gpu_block_pool(gpu_manager.block_pool) - request = Request( - request_id="dcp-draft-source", - prompt_token_ids=list(range(prompt_len)), - sampling_params=SamplingParams(max_tokens=2), - pooling_params=None, - mm_features=None, - block_hasher=get_request_block_hasher(block_size, sha256), - ) - # Follow allocate -> connector metadata -> compute completion, including - # one decode step so the final prefill blocks can be offloaded. - while request.num_computed_tokens <= prompt_len: - if request.num_computed_tokens == prompt_len: - request.append_output_token_ids([prompt_len]) - num_new_tokens = min( - chunk_size, request.num_tokens - request.num_computed_tokens - ) - new_blocks = gpu_manager.allocate_slots( - request, num_new_tokens, num_lookahead_tokens=3 - ) - assert new_blocks is not None - if request.num_computed_tokens == 0: - scheduler.update_state_after_alloc( - request, gpu_manager.get_blocks(request.request_id), 0 - ) - output = make_scheduler_output( - {request.request_id: num_new_tokens}, - new_reqs={request.request_id: new_blocks.get_block_ids()}, - ) - else: - output = make_scheduler_output( - {request.request_id: num_new_tokens}, - cached_req_new_blocks={request.request_id: new_blocks.get_block_ids()}, - ) - store_meta = scheduler.build_connector_meta(output) - request.num_computed_tokens += num_new_tokens - if store_meta.store_event >= 0: - simulate_store_completion(scheduler, store_meta.store_event) - - matching_request = Request( - request_id="dcp-draft-load", - prompt_token_ids=request.prompt_token_ids, - sampling_params=request.sampling_params, - pooling_params=None, - mm_features=None, - block_hasher=request._block_hasher, - ) - expected_hit = ( - 15 if draft_sliding_window and retention_interval is not None else 16 - ) * scheduler_block_size - assert gpu_manager.get_computed_blocks(matching_request)[1] == expected_hit - scheduler.request_finished(request, []) - gpu_manager.free(request) - assert gpu_manager.block_pool.reset_prefix_cache() - - hit_tokens, is_async = scheduler.get_num_new_matched_tokens( - matching_request, num_computed_tokens=0 - ) - assert hit_tokens is not None - assert (hit_tokens, is_async) == (expected_hit, True) - - assert ( - gpu_manager.allocate_slots( - matching_request, - num_new_tokens=0, - num_external_computed_tokens=hit_tokens, - delay_cache_blocks=True, - ) - is not None - ) - load_blocks = gpu_manager.get_blocks(matching_request.request_id) - scheduler.update_state_after_alloc(matching_request, load_blocks, hit_tokens) - load_meta = scheduler.build_connector_meta( - make_scheduler_output( - {matching_request.request_id: 0}, - new_reqs={matching_request.request_id: load_blocks.get_block_ids()}, - ) - ) - # SWA loads only the eight-block tail after null padding; a sharded full - # draft must load every prefix block, with the same DCP geometry as the target. - num_draft_blocks = 8 if draft_sliding_window else hit_tokens // draft_block_size - assert load_meta.load_gpu_blocks == [ - block.block_id - for block in ( - *load_blocks.blocks[0], - *load_blocks.blocks[1][-num_draft_blocks:], - ) - ] - assert ( - len(load_meta.load_gpu_blocks) - == hit_tokens // scheduler_block_size + num_draft_blocks - ) - target_hashes = [ - make_block_hash_with_group_id(matching_request.block_hashes[(i + 1) * 8 - 1], 0) - for i in range(hit_tokens // scheduler_block_size) - ] - draft_hashes = [ - make_block_hash_with_group_id( - matching_request.block_hashes[(i + 1) * draft_block_size // block_size - 1], - 1, - ) - for i in range( - hit_tokens // draft_block_size - num_draft_blocks, - hit_tokens // draft_block_size, - ) - ] - assert [ - scheduler.cpu_block_pool.blocks[bid].block_hash - for bid in load_meta.load_cpu_blocks - ] == target_hashes + draft_hashes diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py index 324f6aeed8..e96b335a5c 100644 --- a/vllm/config/vllm.py +++ b/vllm/config/vllm.py @@ -1143,21 +1143,10 @@ class VllmConfig: self.kv_transfer_config = KVTransferConfig() if kv_offloading_backend == "native": - if envs.VLLM_USE_SIMPLE_KV_OFFLOAD: - config_connector = "SimpleCPUOffloadConnector" - else: - config_connector = "OffloadingConnector" - self.kv_transfer_config.kv_connector = config_connector + self.kv_transfer_config.kv_connector = "OffloadingConnector" self.kv_transfer_config.kv_connector_extra_config.update( {"cpu_bytes_to_use": kv_offloading_size * (1 << 30)} ) - elif kv_offloading_backend == "lmcache": - # Default to LMCache multi-process (MP) mode. The actual KV - # storage capacity is managed by the standalone LMCache server - # process, so ``kv_offloading_size`` is not propagated here. - # ``LMCacheMPConnector`` falls back to ``tcp://localhost:5555`` - # when host/port are not provided via extra_config. - self.kv_transfer_config.kv_connector = "LMCacheMPConnector" # This is the same for all backends self.kv_transfer_config.kv_role = "kv_both" diff --git a/vllm/distributed/ec_transfer/ec_connector/factory.py b/vllm/distributed/ec_transfer/ec_connector/factory.py index 1c33af5513..e9f6d037e2 100644 --- a/vllm/distributed/ec_transfer/ec_connector/factory.py +++ b/vllm/distributed/ec_transfer/ec_connector/factory.py @@ -77,21 +77,3 @@ class ECConnectorFactory: # Register various connectors here. # The registration should not be done in each individual file, as we want to # only load the files corresponding to the current connector. - -ECConnectorFactory.register_connector( - "ECExampleConnector", - "vllm.distributed.ec_transfer.ec_connector.example_connector", - "ECExampleConnector", -) - -ECConnectorFactory.register_connector( - "ECCPUConnector", - "vllm.distributed.ec_transfer.ec_connector.cpu.connector", - "ECCPUConnector", -) - -ECConnectorFactory.register_connector( - "ECMooncakeConnector", - "vllm.distributed.ec_transfer.ec_connector.mooncake_ec_connector", - "ECMooncakeConnector", -) diff --git a/vllm/distributed/kv_transfer/kv_connector/factory.py b/vllm/distributed/kv_transfer/kv_connector/factory.py index ff4c552842..f25f0b1ece 100644 --- a/vllm/distributed/kv_transfer/kv_connector/factory.py +++ b/vllm/distributed/kv_transfer/kv_connector/factory.py @@ -150,48 +150,12 @@ class KVConnectorFactory: # The registration should not be done in each individual file, as we want to # only load the files corresponding to the current connector. -KVConnectorFactory.register_connector( - "ExampleHiddenStatesConnector", - "vllm.distributed.kv_transfer.kv_connector.v1.example_hidden_states_connector", - "ExampleHiddenStatesConnector", -) - -KVConnectorFactory.register_connector( - "NixlConnector", - "vllm.distributed.kv_transfer.kv_connector.v1.nixl", - "NixlConnector", -) - -KVConnectorFactory.register_connector( - "NixlPullConnector", - "vllm.distributed.kv_transfer.kv_connector.v1.nixl", - "NixlPullConnector", -) - -KVConnectorFactory.register_connector( - "NixlPushConnector", - "vllm.distributed.kv_transfer.kv_connector.v1.nixl", - "NixlPushConnector", -) - KVConnectorFactory.register_connector( "MultiConnector", "vllm.distributed.kv_transfer.kv_connector.v1.multi_connector", "MultiConnector", ) -KVConnectorFactory.register_connector( - "HiSparseConnector", - "vllm.distributed.kv_transfer.kv_connector.v1.hisparse.connector", - "HiSparseConnector", -) - -KVConnectorFactory.register_connector( - "MoRIIOConnector", - "vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_connector", - "MoRIIOConnector", -) - KVConnectorFactory.register_connector( "OffloadingConnector", "vllm.distributed.kv_transfer.kv_connector.v1.offloading_connector", @@ -203,14 +167,3 @@ KVConnectorFactory.register_connector( "vllm.distributed.kv_transfer.kv_connector.v1.decode_bench_connector", "DecodeBenchConnector", ) - -KVConnectorFactory.register_connector( - "SimpleCPUOffloadConnector", - "vllm.distributed.kv_transfer.kv_connector.v1.simple_cpu_offload_connector", - "SimpleCPUOffloadConnector", -) -KVConnectorFactory.register_connector( - "HF3FSKVConnector", - "vllm.distributed.kv_transfer.kv_connector.v1.hf3fs.hf3fs_connector", - "HF3FSKVConnector", -) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/hf3fs/__init__.py b/vllm/distributed/kv_transfer/kv_connector/v1/hf3fs/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/hf3fs/hf3fs_client.py b/vllm/distributed/kv_transfer/kv_connector/v1/hf3fs/hf3fs_client.py deleted file mode 100644 index e0bf1dbfc4..0000000000 --- a/vllm/distributed/kv_transfer/kv_connector/v1/hf3fs/hf3fs_client.py +++ /dev/null @@ -1,302 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -import logging -import multiprocessing -import os -import threading -from functools import wraps -from pathlib import Path - -import torch -import torch.utils.cpp_extension -from torch.utils.cpp_extension import load - -root = Path(__file__).parent.resolve() -cuda_include_path = os.path.join(torch.utils.cpp_extension.CUDA_HOME, "include") -hf3fs_utils = load( - name="hf3fs_utils", - sources=[f"{root}/utils/hf3fs_utils.cpp"], - extra_include_paths=[cuda_include_path], -) - -logger = logging.getLogger(__name__) - -HF3FS_AVAILABLE = True -try: - from hf3fs_fuse.io import ( - deregister_fd, - extract_mount_point, - make_ioring, - make_iovec, - register_fd, - ) -except ImportError: - HF3FS_AVAILABLE = False - - -def rsynchronized(): - def _decorator(func): - @wraps(func) - def wrapper(self, *args, **kwargs): - with self.rlock: - return func(self, *args, **kwargs) - - return wrapper - - return _decorator - - -def wsynchronized(): - def _decorator(func): - @wraps(func) - def wrapper(self, *args, **kwargs): - with self.wlock: - return func(self, *args, **kwargs) - - return wrapper - - return _decorator - - -class Hf3fsClient: - def __init__(self, path: str, size: int, bytes_per_page: int, entries: int): - """Initialize the HF3FS client with hf3fs_fuse. - - Args: - path: Path to the file used for storage - size: Total size of the storage file in bytes - bytes_per_page: Size of each page in bytes - entries: Maximum number of concurrent operations - - """ - if not HF3FS_AVAILABLE: - raise ImportError( - "hf3fs_fuse.io is not available. Please install the hf3fs_fuse package." - ) - - self.path = path - self.size = size - self.bytes_per_page = bytes_per_page - self.entries = entries - - self._closed = False - - self.file = None - self.shm_r = None - self.shm_w = None - self.ior_r = None - self.ior_w = None - self.iov_r = None - self.iov_w = None - try: - # Create the file if it doesn't exist and set its size - self.file = os.open(self.path, os.O_RDWR | os.O_CREAT) - os.ftruncate(self.file, size) - register_fd(self.file) - - self.hf3fs_mount_point = extract_mount_point(path) - self.bs = self.bytes_per_page - self.shm_r = multiprocessing.shared_memory.SharedMemory( - size=self.bs * self.entries, create=True - ) - self.shm_w = multiprocessing.shared_memory.SharedMemory( - size=self.bs * self.entries, create=True - ) - - self.shm_r_tensor = torch.frombuffer(self.shm_r.buf, dtype=torch.uint8) - self.shm_w_tensor = torch.frombuffer(self.shm_w.buf, dtype=torch.uint8) - - numel = self.bs * self.entries - self.r_pinned = torch.empty( - numel, - dtype=torch.uint8, - device="cpu", - pin_memory=True, - ) - self.w_pinned = torch.empty( - numel, - dtype=torch.uint8, - device="cpu", - pin_memory=True, - ) - - self.numa = -1 - self.ior_r = make_ioring( - self.hf3fs_mount_point, - self.entries, - for_read=True, - timeout=1, - numa=self.numa, - ) - self.ior_w = make_ioring( - self.hf3fs_mount_point, - self.entries, - for_read=False, - timeout=1, - numa=self.numa, - ) - self.iov_r = make_iovec(self.shm_r, self.hf3fs_mount_point) - self.iov_w = make_iovec(self.shm_w, self.hf3fs_mount_point) - self.shm_r.unlink() - self.shm_w.unlink() - - self.rlock = threading.RLock() - self.wlock = threading.RLock() - - self.stream = torch.cuda.Stream() - self.stream_ptr_int = self.stream.cuda_stream - - except Exception: - self._release_resources() - raise - - logger.debug( - "Initialized HF3FS client with file: %s, size: %s bytes", path, size - ) - - def _release_resources(self) -> None: - """Release all acquired resources safely.""" - # iov must be released before ioring and shm - for attr in ("iov_r", "iov_w", "ior_r", "ior_w"): - obj = getattr(self, attr, None) - if obj is not None: - del obj - setattr(self, attr, None) - - for attr in ("shm_r", "shm_w"): - shm = getattr(self, attr, None) - if shm is not None: - try: - shm.close() - except Exception as e: - logger.warning("Failed to close %s: %s", attr, e) - setattr(self, attr, None) - - if self.file is not None: - try: - deregister_fd(self.file) - except Exception as e: - logger.warning("deregister_fd failed: %s", e) - try: - os.close(self.file) - except OSError as e: - logger.warning("os.close failed: %s", e) - self.file = None - - @rsynchronized() - def batch_read(self, offsets: list[int], tensors: list[torch.Tensor]) -> list[int]: - """Read data from the file at specified offsets into tensors. - - Args: - offsets: List of byte offsets to read from - tensors: List of tensors to read data into - - Returns: - List of operation results (0 for success, non-zero for error) - - """ - self.check(offsets, tensors) - assert self.ior_r is not None - assert self.iov_r is not None - - # prepare - current = 0 - for offset, tensor in zip(offsets, tensors): - size = tensor.numel() * tensor.itemsize - self.ior_r.prepare( - self.iov_r[current : current + size], True, self.file, offset - ) - current += size - - # submit - ionum = len(offsets) - resv = self.ior_r.submit().wait(min_results=ionum) - - # results - with torch.cuda.stream(self.stream): - hf3fs_utils.read_shm( - self.shm_r_tensor, self.r_pinned, tensors, self.stream_ptr_int - ) - results = [res.result for res in resv] - - return results - - @wsynchronized() - def batch_write( - self, offsets: list[int], tensors: list[torch.Tensor], event: torch.cuda.Event - ) -> list[int]: - """Write data from tensors to the file at specified offsets. - - Args: - offsets: List of byte offsets to write to - tensors: List of tensors containing data to write - event: CUDA event to synchronize on before reading the tensors - - Returns: - List of operation results (0 for success, non-zero for error) - - """ - self.check(offsets, tensors) - assert self.ior_w is not None - assert self.iov_w is not None - - # prepare - with torch.cuda.stream(self.stream): - self.stream.wait_event(event) - hf3fs_utils.write_shm( - tensors, self.shm_w_tensor, self.w_pinned, self.stream_ptr_int - ) - - current = 0 - for offset, tensor in zip(offsets, tensors): - size = tensor.numel() * tensor.itemsize - self.ior_w.prepare( - self.iov_w[current : current + size], False, self.file, offset - ) - current += size - - # submit - ionum = len(offsets) - resv = self.ior_w.submit().wait(min_results=ionum) - - # results - results = [res.result for res in resv] - - return results - - def check(self, offsets: list[int], tensors: list[torch.Tensor]) -> None: - sizes = [t.numel() * t.itemsize for t in tensors] - if any( - [ - len(offsets) > self.entries, - len(offsets) != len(sizes), - any( - offset < 0 or offset + size > self.size - for offset, size in zip(offsets, sizes) - ), - any(size > self.bytes_per_page for size in sizes), - ] - ): - self.close() - raise ValueError("Hf3fsClient.check Failed") - - def get_size(self) -> int: - """Get the total size of the storage file. - - Returns: - Size of the file in bytes - - """ - return self.size - - def close(self) -> None: - """Close the client and clean up resources.""" - if self._closed: - return - self._closed = True - self._release_resources() - - def flush(self) -> None: - """Flush any pending writes to disk.""" - if not self._closed and self.file is not None: - os.fsync(self.file) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/hf3fs/hf3fs_connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/hf3fs/hf3fs_connector.py deleted file mode 100644 index 1b37a88440..0000000000 --- a/vllm/distributed/kv_transfer/kv_connector/v1/hf3fs/hf3fs_connector.py +++ /dev/null @@ -1,1196 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""HF3FS KV Connector Implementation for vLLM. - -This module implements a KV connector that uses -the 3FS for storing and retrieving KV cache data. - -Key components: -1. HF3FSConnector: Main connector implementation - 2.1 AsyncOperationManager: Manages async save/load operations with background threads - 2.2 HF3FSConnectorMetadata: Container for connector metadata -3. HF3FSMetadataServer: Mini Metadata server for HF3FS connector -4. HF3FSClient: 3FS Client Implementation -""" - -import atexit -import concurrent -import copy -import hashlib -import os -import queue -import signal -import threading -import time -from concurrent.futures import Future -from dataclasses import dataclass -from queue import Empty -from typing import Any, Optional - -import numpy as np -import torch - -from vllm.config import VllmConfig -from vllm.distributed.kv_transfer.kv_connector.v1.base import ( - KVConnectorBase_V1, - KVConnectorMetadata, - KVConnectorRole, -) -from vllm.distributed.kv_transfer.kv_connector.v1.hf3fs.hf3fs_metadata_server import ( - Hf3fsGlobalMetadataClient as Hf3fsMetadataClient, -) -from vllm.distributed.kv_transfer.kv_connector.v1.hf3fs.utils import ( - gather_scatter_helper, -) -from vllm.distributed.kv_transfer.kv_connector.v1.hf3fs.utils.common import ( - AtomicCounter, - HF3FSConnectorMetadata, - HF3FSRequestMetadata, - LoadBlockInfo, - RequestSchedulingState, -) -from vllm.distributed.kv_transfer.kv_connector.v1.hf3fs.utils.gather_scatter_helper import ( # noqa: E501 - CopyBufferAllocator, -) -from vllm.distributed.kv_transfer.kv_connector.v1.metrics import ( - KVConnectorPromMetrics, - KVConnectorStats, - PromMetric, - PromMetricT, -) -from vllm.distributed.parallel_state import get_tensor_model_parallel_rank -from vllm.forward_context import ForwardContext -from vllm.logger import init_logger -from vllm.v1.attention.backend import AttentionMetadata -from vllm.v1.core.kv_cache_manager import KVCacheBlocks -from vllm.v1.core.sched.output import SchedulerOutput -from vllm.v1.kv_cache_interface import KVCacheConfig -from vllm.v1.metrics.utils import create_metric_per_engine -from vllm.v1.request import Request - -HF3FS_AVAILABLE = True -Hf3fsClient = None -try: - from hf3fs_fuse.io import deregister_fd # noqa: F401 - - from vllm.distributed.kv_transfer.kv_connector.v1.hf3fs.hf3fs_client import ( - Hf3fsClient as _RealClient, - ) - - Hf3fsClient = _RealClient -except Exception: - HF3FS_AVAILABLE = False - from vllm.distributed.kv_transfer.kv_connector.v1.hf3fs.utils.hf3fs_mock_client import ( # noqa: E501 - Hf3fsClient as _MockClient, - ) - - Hf3fsClient = _MockClient # type: ignore - -# Constants -DEFAULT_MAX_IO_ENTRIES = 8 - -logger = init_logger(__name__) - - -# ============================================================================ -# Async Operation Management -# ============================================================================ - - -class AsyncOperationManager: - """Manages async save/load operations with background threads.""" - - def __init__(self, connector: "HF3FSKVConnector"): - # Store connector reference and extract commonly used attributes - self._connector = connector - self._device = connector._device - self._dtype = connector._dtype - self._shape_per_page = connector._shape_per_page - self._bytes_per_page = connector._bytes_per_page - self._rank = connector._rank - self._numjobs = connector._numjobs - self._max_device_buffer_count = connector._max_device_buffer_count - - # Operation tracking - self._save_futures: dict[str, list[Future]] = {} - self._load_futures: dict[str, Future] = {} - self._pending_finished_requests: set[str] = set() - - # Initialize resources - self._init_cuda_resources() - self._init_worker_threads() - - # Metrics - self.hf3fs_stats = HF3FSKVConnectorStats() - - logger.info("AsyncOperationManager initialized for rank %d", self._rank) - - def _init_cuda_resources(self) -> None: - """Initialize CUDA streams, events and buffer allocators.""" - # CUDA streams for async operations - self._save_stream = torch.cuda.Stream() - self._load_stream = torch.cuda.Stream() - self._save_event = torch.cuda.Event() - - # Buffer allocators for data copying - self._save_buffer_allocator = CopyBufferAllocator( - self._device, - self._dtype, - self._shape_per_page, - self._max_device_buffer_count, - ) - self._load_buffer_allocator = CopyBufferAllocator( - self._device, - self._dtype, - self._shape_per_page, - self._max_device_buffer_count, - ) - - def _init_worker_threads(self) -> None: - """Initialize worker threads and I/O executor.""" - # Thread synchronization - self._stop_event = threading.Event() - self._save_queue: queue.Queue[Any] = queue.Queue() - self._load_queue: queue.Queue[Any] = queue.Queue() - - # I/O thread pool - self._io_executor = concurrent.futures.ThreadPoolExecutor( - max_workers=self._numjobs, - thread_name_prefix=f"HF3FS-Rank{self._rank}", - ) - - # Background worker threads - self._save_thread = threading.Thread(target=self._save_worker, daemon=True) - self._load_thread = threading.Thread(target=self._load_worker, daemon=True) - self._save_thread.start() - self._load_thread.start() - - def submit_save_operation(self, request_id: str, block_ids, block_hashes) -> Future: - """Submit a save operation for async execution.""" - future: Future[Any] = Future() - main_stream_event = torch.cuda.Event() - main_stream_event.record() - task = (request_id, block_ids, block_hashes, future, main_stream_event) - self._save_queue.put(task) - - if request_id not in self._save_futures: - self._save_futures[request_id] = [] - self._save_futures[request_id].append(future) - return future - - def submit_load_operation(self, request_id: str, block_ids, block_hashes) -> Future: - """Submit a load operation for async execution.""" - future: Future[Any] = Future() - task = (request_id, block_ids, block_hashes, future) - self._load_queue.put(task) - self._load_futures[request_id] = future - return future - - def get_finished_operations( - self, finished_req_ids: set[str] - ) -> tuple[set[str], set[str]]: - completed_saves = self._check_completed_saves(finished_req_ids) - completed_loads = self._check_completed_loads() - - if completed_saves or completed_loads: - logger.info( - "HF3FS Connector Completed: %d saves, %d loads operations", - len(completed_saves), - len(completed_loads), - ) - - return completed_saves, completed_loads - - def _check_completed_saves(self, finished_req_ids: set[str]) -> set[str]: - """Check for completed save operations.""" - completed = set() - - # Check pending finished requests first - for request_id in list(self._pending_finished_requests): - if request_id in self._save_futures and self._all_saves_done(request_id): - completed.add(request_id) - self._save_futures.pop(request_id) - self._pending_finished_requests.remove(request_id) - - # Process newly finished requests - for request_id in finished_req_ids: - if request_id in self._save_futures: - if self._all_saves_done(request_id): - completed.add(request_id) - self._save_futures.pop(request_id) - else: - self._pending_finished_requests.add(request_id) - else: - completed.add(request_id) - - return completed - - def _check_completed_loads(self) -> set[str]: - """Check for completed load operations.""" - completed = set() - for request_id in list(self._load_futures): - if self._load_futures[request_id].done(): - completed.add(request_id) - self._load_futures.pop(request_id) - return completed - - def _all_saves_done(self, request_id: str) -> bool: - """Check if all save operations for a request are completed.""" - return all(future.done() for future in self._save_futures[request_id]) - - def _save_worker(self) -> None: - """Background worker for handling save operations.""" - torch.accelerator.set_device_index(self._device) - while not self._stop_event.is_set(): - try: - task = self._save_queue.get(block=True, timeout=1) - self._handle_save_task(task) - except Empty: - continue - except Exception as e: - logger.error("Save worker error: %s", e) - - def _load_worker(self) -> None: - """Background worker for handling load operations.""" - torch.accelerator.set_device_index(self._device) - while not self._stop_event.is_set(): - try: - task = self._load_queue.get(block=True, timeout=1) - self._handle_load_task(task) - except Empty: - continue - except Exception as e: - logger.error("Load worker error: %s", e) - - def _handle_save_task(self, task) -> None: - """Handle individual save task with proper stream synchronization.""" - request_id, block_ids, block_hashes, future, main_stream_event = task - start_time = time.perf_counter() - buffers = None - try: - # Step1: Allocate storage pages - key_pairs = [(hash_val, "") for hash_val in block_hashes] - allocation_results = ( - self._connector._metadata_client.allocate_pages_for_keys( - self._rank, key_pairs - ) - ) - - if any(result[1] < 0 for result in allocation_results): - return self._fail_task( - "Saved", "Page allocation failed", request_id, future - ) - - page_indices = [result[1] for result in allocation_results] - offsets = [idx * self._bytes_per_page for idx in page_indices] - - # Step2: Allocate buffers and gather KV cache data - buffers = self._save_buffer_allocator.alloc_buffer(len(block_ids)) - if buffers is None: - return self._fail_task( - "Saved", - f"Buffer allocation failed for {len(block_ids)} blocks", - request_id, - future, - ) - - # Synchronize streams and gather data - with torch.cuda.stream(self._save_stream): - self._save_stream.wait_event(main_stream_event) # Wait for main stream - self._connector._gather_or_scatter_kv_caches( - block_ids, buffers, "gather" - ) - - save_stream_event = torch.cuda.Event() - save_stream_event.record(self._save_stream) # Record gather completion - - # Step3: Write data in batches - write_futures = [] - for i in range(0, len(offsets), DEFAULT_MAX_IO_ENTRIES): - batch_offsets = offsets[i : i + DEFAULT_MAX_IO_ENTRIES] - batch_buffers = buffers[i : i + DEFAULT_MAX_IO_ENTRIES] - client = self._connector._clients[self._connector._ac.next()] - write_future = self._io_executor.submit( - client.batch_write, batch_offsets, batch_buffers, save_stream_event - ) - write_futures.append(write_future) - - # Check write results - write_success = all( - result == self._bytes_per_page - for write_future in write_futures - for result in write_future.result() - ) - - # Step4: Confirm writes to metadata server - if write_success: - written_keys = list(zip(block_hashes, page_indices)) - self._connector._metadata_client.confirm_write_for_keys( - self._rank, written_keys, [] - ) - self._save_buffer_allocator.free_buffer(buffers) - return self._succeed_task( - "Saved", start_time, request_id, len(block_ids), future - ) - else: - self._connector._metadata_client.confirm_write_for_keys( - self._rank, [], page_indices - ) - self._save_buffer_allocator.free_buffer(buffers) - return self._fail_task( - "Saved", "Write operation failed", request_id, future - ) - - except Exception as e: - if buffers is not None: - self._save_buffer_allocator.free_buffer(buffers) - return self._fail_task( - "Saved", f"Task execution error: {e}", request_id, future - ) - - def _handle_load_task(self, task) -> None: - """Handle individual load task.""" - request_id, block_ids, block_hashes, future = task - start_time = time.perf_counter() - buffers = None - try: - # Step1: Get block locations from metadata server - page_indices = self._connector._metadata_client.get_key_locations( - self._rank, block_hashes - ) - - if any(idx is None for idx in page_indices): - return self._fail_task("Loaded", "Blocks not found", request_id, future) - - # Allocate read buffer - buffers = self._load_buffer_allocator.alloc_buffer(len(block_ids)) - if buffers is None: - return self._fail_task( - "Loaded", - f"Buffer allocation failed for {len(block_ids)} blocks", - request_id, - future, - ) - - # Step2: Read data in batches - offsets = [idx * self._bytes_per_page for idx in page_indices] - read_futures = [] - for i in range(0, len(offsets), DEFAULT_MAX_IO_ENTRIES): - batch_offsets = offsets[i : i + DEFAULT_MAX_IO_ENTRIES] - batch_buffers = buffers[i : i + DEFAULT_MAX_IO_ENTRIES] - client = self._connector._clients[self._connector._ac.next()] - read_future = self._io_executor.submit( - client.batch_read, batch_offsets, batch_buffers - ) - read_futures.append(read_future) - - # Check read results - read_success = all( - result == self._bytes_per_page - for read_future in read_futures - for result in read_future.result() - ) - - if not read_success: - self._load_buffer_allocator.free_buffer(buffers) - return self._fail_task( - "Loaded", "Read operation failed", request_id, future - ) - - # Step3: Scatter data back to KV cache - with torch.cuda.stream(self._load_stream): - self._connector._gather_or_scatter_kv_caches( - block_ids, buffers, "scatter" - ) - - self._load_stream.synchronize() - self._load_buffer_allocator.free_buffer(buffers) - return self._succeed_task( - "Loaded", start_time, request_id, len(block_ids), future - ) - - except Exception as e: - if buffers is not None: - self._load_buffer_allocator.free_buffer(buffers) - return self._fail_task( - "Loaded", f"Task execution error: {e}", request_id, future - ) - - def _fail_task( - self, operation: str, error_msg: str, request_id: str, future: Future - ) -> None: - """Helper to fail task with error logging.""" - logger.error( - "%s for %s request %s", - error_msg, - operation, - request_id, - ) - self.hf3fs_stats.record_failed_task_count(operation) - future.set_result(False) - - def _succeed_task( - self, - operation: str, - start_time: float, - request_id: str, - block_count: int, - future: Future, - ) -> None: - """Helper to succeed task with logging.""" - duration = time.perf_counter() - start_time - logger.info( - "%s %s: %d blocks in %.2fs", - operation, - request_id, - block_count, - duration, - ) - self.hf3fs_stats.record_success_task_duration(operation, duration) - future.set_result(True) - - def shutdown(self) -> None: - """Clean shutdown of all background threads and resources.""" - self._stop_event.set() - self._save_thread.join() - self._load_thread.join() - self._io_executor.shutdown(wait=True) - logger.info("AsyncOperationManager shutdown completed") - - -# ============================================================================ -# HF3FS Connector -# ============================================================================ - - -class HF3FSKVConnector(KVConnectorBase_V1): - """HF3FS KV Connector implementation.""" - - def __init__( - self, - vllm_config: "VllmConfig", - role: KVConnectorRole, - kv_cache_config: "KVCacheConfig", - ): - super().__init__( - vllm_config=vllm_config, role=role, kv_cache_config=kv_cache_config - ) - - # Core configuration - self._vllm_config = vllm_config - self._role = role - self._block_size = vllm_config.cache_config.block_size - self._use_mla = vllm_config.model_config.use_mla - self._model_config = vllm_config.model_config - - logger.info("Using MLA: %s", self._use_mla) - - # HF3FS configuration - kv_config = vllm_config.kv_transfer_config - assert kv_config is not None - - self._storage_path = kv_config.get_from_extra_config( - "hf3fs_storage_path", "/vllm-workspace/mnt/hf3fs" - ) - self._metadata_server_url = kv_config.get_from_extra_config( - "hf3fs_metadata_server_url", "http://localhost:18000" - ) - self._file_size = kv_config.get_from_extra_config( - "hf3fs_file_size", 1024 * 1024 * 1024 - ) - self._numjobs = kv_config.get_from_extra_config("hf3fs_client_numjobs", 16) - self._max_device_buffer_count = kv_config.get_from_extra_config( - "hf3fs_max_device_buffer_count", 128 - ) - self._max_device_buffer_count = max( - self._max_device_buffer_count, self._numjobs * DEFAULT_MAX_IO_ENTRIES - ) - - if self._role == KVConnectorRole.SCHEDULER: - self._scheduling_states: dict[str, RequestSchedulingState] = {} - self._metadata_client = Hf3fsMetadataClient() - self._metadata_client.initialize(0, role="scheduler") - - atexit.register(self.close) - signal.signal(signal.SIGINT, lambda sig, frame: self.close()) - signal.signal(signal.SIGTERM, lambda sig, frame: self.close()) - signal.signal(signal.SIGQUIT, lambda sig, frame: self.close()) - - logger.info( - "HF3FSKVConnector initialized: path=%s, role=%s", - self._storage_path, - self._role.name, - ) - - ############################################################ - # Worker Side Methods - ############################################################ - - def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]) -> None: - self._kv_caches = kv_caches - self._setup_kv_cache_config() - self._setup_storage_clients() - self._async_manager = AsyncOperationManager(self) - - def _setup_kv_cache_config(self): - first_cache = next(iter(self._kv_caches.values())) - self._device = first_cache.device - self._dtype = first_cache.dtype - element_size = first_cache.element_size() - - # Standardized per-layer [B, H, N, C] view; MLA is just H == 1 with the - # latent vector as the content dim. - assert len(first_cache.shape) == 4, ( - f"expected a [B, H, N, C] KV cache view, got {tuple(first_cache.shape)}" - ) - num_blocks, num_heads, block_size, content_dim = first_cache.shape - cache_strides = first_cache.stride() - assert all( - cache.shape == first_cache.shape and cache.stride() == cache_strides - for cache in self._kv_caches.values() - ), "HF3FS requires uniform KV cache shapes and strides." - - self._local_total_tokens = num_blocks * block_size - self._local_block_size = block_size - self._num_heads = num_heads - self._content_dim = content_dim - self._kv_cache_strides = cache_strides - - layer_block_size = num_heads * block_size * content_dim * element_size - self._bytes_per_page = layer_block_size * len(self._kv_caches) - self._shape_per_page = [ - len(self._kv_caches), - num_heads, - block_size, - content_dim, - ] - - self._kvcache_ptrs = torch.tensor( - [cache.data_ptr() for cache in self._kv_caches.values()], - dtype=torch.int64, - device=self._device, - ) - - def _setup_storage_clients(self): - os.makedirs(self._storage_path, exist_ok=True) - - self._rank = get_tensor_model_parallel_rank() - file_path = os.path.join( - self._storage_path, f"hf3fs_vllm_data_file_{self._rank}" - ) - - try: - # Initialize HF3FS clients - self._ac = AtomicCounter(self._numjobs) - assert Hf3fsClient is not None - self._clients = [ - Hf3fsClient( - path=file_path, - size=self._file_size, - bytes_per_page=self._bytes_per_page, - entries=DEFAULT_MAX_IO_ENTRIES, - ) - for _ in range(self._numjobs) - ] - - # Initialize metadata client - num_pages = self._file_size // self._bytes_per_page - self._metadata_client = Hf3fsMetadataClient() - self._metadata_client.initialize(self._rank, num_pages, role="worker") - except Exception as e: - logger.error("HF3FS client initialization failed: %s", e) - raise - - def save_kv_layer( - self, - layer_name: str, - kv_layer: torch.Tensor, - attn_metadata: "AttentionMetadata", - **kwargs, - ) -> None: - """HF3FSConnector does not do layerwise saving.""" - pass - - def wait_for_save(self) -> None: - metadata = self._get_connector_metadata() - if not isinstance(metadata, HF3FSConnectorMetadata): - logger.error("Invalid metadata type: %s", type(metadata)) - return - - for request in metadata.requests: - if request.save_block_op is None: - continue - - skip_blocks = request.save_block_op.skip_leading_blocks - block_hashes = self._generate_block_hashes(request.token_ids, skip_blocks) - block_ids = request.block_ids[skip_blocks : skip_blocks + len(block_hashes)] - - for i in range(0, len(block_ids), self._max_device_buffer_count): - batch_block_ids = block_ids[i : i + self._max_device_buffer_count] - batch_block_hashes = block_hashes[i : i + self._max_device_buffer_count] - self._async_manager.submit_save_operation( - request.request_id, batch_block_ids, batch_block_hashes - ) - - def start_load_kv(self, forward_context: "ForwardContext", **kwargs) -> None: - metadata = self._get_connector_metadata() - if not isinstance(metadata, HF3FSConnectorMetadata): - logger.error("Invalid metadata type for loading") - return - - for request in metadata.requests: - if request.load_block_op is None: - continue - - load_op = request.load_block_op - block_ids = request.block_ids[: load_op.num_blocks_to_load] - block_hashes = self._generate_block_hashes( - request.token_ids, load_op.num_computed_blocks, len(block_ids) - ) - - for i in range(0, len(block_ids), self._max_device_buffer_count): - batch_block_ids = block_ids[i : i + self._max_device_buffer_count] - batch_block_hashes = block_hashes[i : i + self._max_device_buffer_count] - self._async_manager.submit_load_operation( - request.request_id, batch_block_ids, batch_block_hashes - ) - - def wait_for_layer_load(self, layer_name: str) -> None: - pass - - def get_finished( - self, finished_req_ids: set[str] - ) -> tuple[set[str] | None, set[str] | None]: - return self._async_manager.get_finished_operations(finished_req_ids) - - def get_kv_connector_stats(self) -> Optional["KVConnectorStats"]: - """Get the KV connector stats collected during the last interval.""" - # Clear stats for next iteration - if ( - hasattr(self, "_async_manager") - and not self._async_manager.hf3fs_stats.is_empty() - ): - return self._async_manager.hf3fs_stats.clone_and_reset() - return None - - ############################################################ - # Scheduler Side Methods - ############################################################ - - def request_finished( - self, - request: "Request", - block_ids: list[int], - ) -> tuple[bool, dict[str, Any] | None]: - return True, None - - def get_num_new_matched_tokens( - self, request: "Request", num_computed_tokens: int - ) -> tuple[int, bool]: - """Get number of new tokens that can be loaded from external cache.""" - try: - state = self._get_or_create_scheduling_state(request.request_id) - state.request = request - assert request.prompt_token_ids is not None - - num_tokens_to_check = self._align_to_block_size( - len(request.prompt_token_ids) - 1 - ) - - if num_tokens_to_check <= num_computed_tokens: - state.load_op = LoadBlockInfo( - num_computed_blocks=num_computed_tokens // self._block_size, - num_blocks_to_load=0, - need_fetch_block_ids=[], - ) - return 0, False - - token_ids_to_check = request.prompt_token_ids[:num_tokens_to_check] - block_hashes = self._generate_block_hashes(token_ids_to_check, 0) - - # Check existence - exists_results = self._metadata_client.batch_key_exists(block_hashes) - - # Count consecutive matches - matched_blocks = next( - (i for i, exists in enumerate(exists_results) if not exists), - len(exists_results), - ) - matched_tokens = matched_blocks * self._block_size - new_hit_tokens = max(0, matched_tokens - num_computed_tokens) - - # Store load operation - state.load_op = LoadBlockInfo( - num_computed_blocks=num_computed_tokens // self._block_size, - num_blocks_to_load=new_hit_tokens // self._block_size, - need_fetch_block_ids=[], - ) - - logger.info( - ( - "Token matching for %s: " - "%d matched (%d blocks), " - "%d new hits, " - "prompt len %d" - ), - request.request_id, - matched_tokens, - matched_blocks, - new_hit_tokens, - len(request.prompt_token_ids), - ) - return new_hit_tokens, new_hit_tokens > 0 - - except Exception as e: - logger.error( - "Error calculating matches for request %s: %s", request.request_id, e - ) - return 0, False - - def update_state_after_alloc( - self, request: "Request", blocks: "KVCacheBlocks", num_external_tokens: int - ) -> None: - """Update state after block allocation.""" - state = self._get_or_create_scheduling_state(request.request_id) - state.request = request - - if num_external_tokens <= 0 or not state.needs_loading(): - return - - # Validate block allocation - assert state.load_op is not None - expected_blocks = state.load_op.num_blocks_to_load - actual_blocks = num_external_tokens // self._block_size - assert actual_blocks == expected_blocks, ( - f"Block count mismatch for {request.request_id}: " - f"expected {expected_blocks}, got {actual_blocks}" - ) - - # Update load operation with allocated block IDs - if actual_blocks > 0: - local_block_ids = blocks.get_unhashed_block_ids() - state.load_op.need_fetch_block_ids.extend(local_block_ids) - state.phase = "WAITING_TO_LOAD" - - def build_connector_meta( - self, scheduler_output: SchedulerOutput - ) -> KVConnectorMetadata: - """Build connector metadata for scheduling step.""" - metadata = HF3FSConnectorMetadata() - - for request_id in scheduler_output.finished_req_ids: - self._scheduling_states.pop(request_id, None) - - # Process requests by phase - self._process_waiting_to_load_requests(metadata) - self._process_new_requests(scheduler_output, metadata) - self._process_cached_requests(scheduler_output, metadata) - - return metadata - - def _process_waiting_to_load_requests( - self, metadata: HF3FSConnectorMetadata - ) -> None: - """Process requests waiting to load.""" - for state in list(self._scheduling_states.values()): - if not state.is_ready_to_load(): - continue - assert state.load_op is not None - assert ( - state.request is not None and state.request.prompt_token_ids is not None - ) - # Create load request metadata - num_cached_blocks = ( - state.load_op.num_computed_blocks + state.load_op.num_blocks_to_load - ) - num_tokens_to_compute = num_cached_blocks * self._block_size - - # Initialize token_ids and allocated_block_ids for loading - state.token_ids = state.request.prompt_token_ids[ - :num_tokens_to_compute - ].copy() - state.allocated_block_ids = state.load_op.need_fetch_block_ids.copy() - - request_metadata = HF3FSRequestMetadata.from_scheduling_state( - state, self._block_size, state.load_op, num_cached_blocks - ) - - if request_metadata: - metadata.add_request(request_metadata) - state.phase = "ACTIVE" - - def _process_new_requests( - self, scheduler_output: SchedulerOutput, metadata: HF3FSConnectorMetadata - ) -> None: - """Process new requests.""" - for request in scheduler_output.scheduled_new_reqs: - state = self._get_or_create_scheduling_state(request.req_id) - - # Calculate tokens to compute - num_tokens_to_compute = ( - request.num_computed_tokens - + scheduler_output.num_scheduled_tokens[request.req_id] - ) - self._initialize_state_from_new_request( - state, request, num_tokens_to_compute - ) - - # Create save metadata (skip cached blocks if any) - num_cached_blocks = None - if state.load_op: - num_cached_blocks = ( - state.load_op.num_computed_blocks + state.load_op.num_blocks_to_load - ) - - request_metadata = HF3FSRequestMetadata.from_scheduling_state( - state, self._block_size, None, num_cached_blocks - ) - - if request_metadata: - metadata.add_request(request_metadata) - state.phase = "ACTIVE" - - def _process_cached_requests( - self, scheduler_output: SchedulerOutput, metadata: HF3FSConnectorMetadata - ) -> None: - """Process cached requests.""" - cached_reqs = scheduler_output.scheduled_cached_reqs - for i, request_id in enumerate(cached_reqs.req_ids): - state = self._get_or_create_scheduling_state(request_id) - assert state.request is not None - - # Update with new tokens and blocks - num_new_tokens = scheduler_output.num_scheduled_tokens[request_id] - num_current_tokens = len(state.token_ids) - new_token_ids = state.request.all_token_ids[ - num_current_tokens : num_current_tokens + num_new_tokens - ] - new_block_ids = cached_reqs.new_block_ids[i] - - state.update_tokens_and_blocks(new_token_ids, new_block_ids) - - # Create save metadata - request_metadata = HF3FSRequestMetadata.from_scheduling_state( - state, self._block_size, None - ) - - if request_metadata: - metadata.add_request(request_metadata) - - @classmethod - def build_kv_connector_stats( - cls, data: dict[str, Any] | None = None - ) -> Optional["KVConnectorStats"]: - """KVConnectorStats resolution method. This method allows dynamically - registered connectors to return their own KVConnectorStats object, - which can implement custom aggregation logic on the data dict. - """ - return ( - HF3FSKVConnectorStats(data=data) - if data is not None - else HF3FSKVConnectorStats() - ) - - @classmethod - def build_prom_metrics( - cls, - vllm_config: VllmConfig, - metric_types: dict[type[PromMetric], type[PromMetricT]], - labelnames: list[str], - per_engine_labelvalues: dict[int, list[object]], - ) -> KVConnectorPromMetrics: - return HF3FSPromMetrics( - vllm_config, metric_types, labelnames, per_engine_labelvalues - ) - - def close(self) -> None: - try: - if hasattr(self, "_async_manager"): - self._async_manager.shutdown() - - if hasattr(self, "_clients"): - for client in self._clients: - client.close() - logger.info("HF3FS clients closed") - except Exception as e: - logger.error("Connector shutdown error: %s", e) - - ############################################################ - # Utility Methods - ############################################################ - - def _get_or_create_scheduling_state( - self, request_id: str - ) -> RequestSchedulingState: - """Get existing or create new scheduling state.""" - if request_id not in self._scheduling_states: - self._scheduling_states[request_id] = RequestSchedulingState( - request_id=request_id - ) - return self._scheduling_states[request_id] - - def _initialize_state_from_new_request( - self, state: RequestSchedulingState, request, num_tokens_to_compute: int - ) -> None: - """Initialize state from new request data.""" - # Handle different block_ids formats in vLLM 0.9.0+ - if isinstance(request.block_ids[0], list): - unfolded_block_ids = request.block_ids[0].copy() - else: - unfolded_block_ids = request.block_ids.copy() - - state.token_ids = request.prompt_token_ids[:num_tokens_to_compute].copy() - state.allocated_block_ids = unfolded_block_ids - state.num_saved_blocks = 0 - - def _generate_block_hashes( - self, - token_ids: list[int], - start_block_id: int, - max_blocks_count: int | None = None, - ) -> list[str]: - """Generate block hashes for token sequence.""" - block_hashes = [] - previous_hash = "" - - for start_idx in range(0, len(token_ids), self._block_size): - if start_idx + self._block_size > len(token_ids): - break - - end_idx = start_idx + self._block_size - block_hash = self._compute_prefix_hash( - token_ids[start_idx:end_idx], previous_hash - ) - - block_index = start_idx // self._block_size - if block_index >= start_block_id: - block_hashes.append(block_hash) - - if max_blocks_count and len(block_hashes) >= max_blocks_count: - break - previous_hash = block_hash - - return block_hashes - - def _gather_or_scatter_kv_caches( - self, block_ids: list[int], block_buffers, operation: str - ): - for buffer_tensor, block_id in zip(block_buffers, block_ids): - start_idx = block_id * self._local_block_size - token_indices = list(range(start_idx, start_idx + self._local_block_size)) - if operation == "gather": - gather_scatter_helper.gather_kv_caches( - self._kvcache_ptrs, - self._local_total_tokens, - buffer_tensor, - token_indices, - self._local_block_size, - self._num_heads, - self._content_dim, - self._kv_cache_strides, - ) - else: - gather_scatter_helper.scatter_kv_caches( - self._kvcache_ptrs, - self._local_total_tokens, - buffer_tensor, - token_indices, - self._local_block_size, - self._num_heads, - self._content_dim, - self._kv_cache_strides, - ) - - def _compute_prefix_hash( - self, token_ids: list[int], previous_hash: str = "" - ) -> str: - """Compute prefix hash for token block.""" - combined_string = f"{previous_hash}_{token_ids}" - return hashlib.md5(combined_string.encode()).hexdigest() - - def _align_to_block_size(self, num_tokens: int) -> int: - """Align token count to block size.""" - return (num_tokens // self._block_size) * self._block_size - - -@dataclass -class HF3FSKVConnectorStats(KVConnectorStats): - """Container for transfer performance metrics.""" - - def __post_init__(self): - if not self.data: - # Empty container init, no data is passed in. - self.reset() - - def reset(self): - # Must be serializable - self.data: dict[str, Any] = { - "save_duration": [], - "load_duration": [], - "num_failed_save": 0, - "num_failed_load": 0, - "num_transfer_task": 0, - } - - def aggregate(self, other: "KVConnectorStats") -> "KVConnectorStats": - if not other.is_empty(): - for k, v in other.data.items(): - accumulator = self.data[k] - if isinstance(accumulator, list): - accumulator.extend(v) - else: # int - self.data[k] += v - return self - - def reduce(self) -> dict[str, int | float]: - # Compute compact representative stats suitable for CLI logging - if self.is_empty(): - return { - "Num transfers task": 0, - "Num save task success": 0, - "Num save task failed": 0, - "Num load task success": 0, - "Num load task failed": 0, - "Avg save duration (ms)": 0, - "P90 save duration (ms)": 0, - "Avg load duration (ms)": 0, - "P90 load duration (ms)": 0, - } - num_success_save = len(self.data["save_duration"] or []) - num_success_load = len(self.data["load_duration"] or []) - num_failed_save = self.data["num_failed_save"] - num_failed_load = self.data["num_failed_load"] - if num_success_save == 0: - save_duration = np.zeros(1) - else: - save_duration = np.asarray(self.data["save_duration"]) - if num_success_load == 0: - load_duration = np.zeros(1) - else: - load_duration = np.asarray(self.data["load_duration"]) - - return { - "Num transfers task": self.data["num_transfer_task"], - "Num save task success": num_success_save, - "Num save task failed": num_failed_save, - "Num load task success": num_success_load, - "Num load task failed": num_failed_load, - "Avg save duration (ms)": round(save_duration.mean().item() * 1e3, 3), - "P90 save duration (ms)": round( - np.percentile(save_duration, 90).item() * 1e3, 3 - ), - "Avg load duration (ms)": round(load_duration.mean().item() * 1e3, 3), - "P90 load duration (ms)": round( - np.percentile(load_duration, 90).item() * 1e3, 3 - ), - } - - def is_empty(self) -> bool: - return self.data["num_transfer_task"] == 0 - - def record_success_task_duration(self, operation, duration): - if operation == "Saved": - self.data["save_duration"].append(duration) - elif operation == "Loaded": - self.data["load_duration"].append(duration) - self.data["num_transfer_task"] += 1 - - def record_failed_task_count(self, operation): - if operation == "Saved": - self.data["num_failed_save"] += 1 - elif operation == "Loaded": - self.data["num_failed_load"] += 1 - self.data["num_transfer_task"] += 1 - - def clone_and_reset(self): - old = copy.copy(self) - self.reset() - return old - - -class HF3FSPromMetrics(KVConnectorPromMetrics): - def __init__( - self, - vllm_config: VllmConfig, - metric_types: dict[type[PromMetric], type[PromMetricT]], - labelnames: list[str], - per_engine_labelvalues: dict[int, list[object]], - ): - super().__init__(vllm_config, metric_types, labelnames, per_engine_labelvalues) - buckets = [ - 0.001, - 0.005, - 0.01, - 0.025, - 0.05, - 0.075, - 0.1, - 0.2, - 0.3, - 0.5, - 0.75, - 1.0, - 5.0, - ] - hf3fs_save_duration = self._histogram_cls( - name="vllm:hf3fs_save_duration_seconds", - documentation="Histogram of save duration for HF3FSKVConnector.", - buckets=buckets, - labelnames=labelnames, - ) - self.hf3fs_save_duration = create_metric_per_engine( - hf3fs_save_duration, self.per_engine_labelvalues - ) - - hf3fs_load_duration = self._histogram_cls( - name="vllm:hf3fs_load_duration_seconds", - documentation="Histogram of load duration for HF3FSKVConnector.", - buckets=buckets, - labelnames=labelnames, - ) - self.hf3fs_load_duration = create_metric_per_engine( - hf3fs_load_duration, self.per_engine_labelvalues - ) - - hf3fs_num_failed_save = self._counter_cls( - name="vllm:hf3fs_num_failed_save", - documentation="Number of failed HF3FS KV save.", - labelnames=labelnames, - ) - self.hf3fs_num_failed_save = create_metric_per_engine( - hf3fs_num_failed_save, self.per_engine_labelvalues - ) - - hf3fs_num_failed_load = self._counter_cls( - name="vllm:hf3fs_num_failed_load", - documentation="Number of failed HF3FS KV load.", - labelnames=labelnames, - ) - self.hf3fs_num_failed_load = create_metric_per_engine( - hf3fs_num_failed_load, self.per_engine_labelvalues - ) - - def observe(self, transfer_stats_data: dict[str, Any], engine_idx: int = 0): - for prom_obj, list_item_key in zip( - [ - self.hf3fs_save_duration, - self.hf3fs_load_duration, - ], - [ - "save_duration", - "load_duration", - ], - ): - for list_item in transfer_stats_data[list_item_key]: - prom_obj[engine_idx].observe(list_item) - for counter_obj, counter_item_key in zip( - [ - self.hf3fs_num_failed_save, - self.hf3fs_num_failed_load, - ], - [ - "num_failed_save", - "num_failed_load", - ], - ): - counter_obj[engine_idx].inc(transfer_stats_data[counter_item_key]) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/hf3fs/hf3fs_metadata_server.py b/vllm/distributed/kv_transfer/kv_connector/v1/hf3fs/hf3fs_metadata_server.py deleted file mode 100644 index 92cf08e5be..0000000000 --- a/vllm/distributed/kv_transfer/kv_connector/v1/hf3fs/hf3fs_metadata_server.py +++ /dev/null @@ -1,532 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""HF3FS Metadata Server with key-based organization.""" - -import argparse -import logging -import threading -from abc import ABC, abstractmethod -from dataclasses import dataclass - -try: - import orjson - - HAS_ORJSON = True -except ImportError: - import json as orjson # type: ignore - - HAS_ORJSON = False - -import requests -from fastapi import FastAPI, HTTPException, Request, Response -from fastapi.responses import ORJSONResponse -from requests.adapters import HTTPAdapter -from urllib3.util.retry import Retry - -logging.basicConfig( - level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s" -) -logger = logging.getLogger(__name__) - - -@dataclass -class RankFileMetadata: - """Manages file page allocation for a single rank.""" - - rank_id: int - num_pages: int - free_pages: list[int] - - def allocate_pages(self, num_pages: int) -> list[int]: - """Allocate specified number of free pages.""" - if len(self.free_pages) < num_pages: - return [] - - allocated = self.free_pages[:num_pages] - self.free_pages = self.free_pages[num_pages:] - return allocated - - def release_pages(self, page_indices: list[int]) -> None: - """Release pages back to free pool.""" - for page_idx in page_indices: - if page_idx not in self.free_pages: - self.free_pages.append(page_idx) - - def get_free_page_count(self) -> int: - """Get current number of free pages.""" - return len(self.free_pages) - - -@dataclass -class KeyMetadata: - """Manages metadata for a single key across multiple ranks.""" - - key: str - rank_to_page: dict[int, int] # rank -> allocated page index - tp_world_size: int - - def add_rank_page(self, rank: int, page_index: int) -> None: - """Add page allocation for a specific rank.""" - self.rank_to_page[rank] = page_index - - def get_all_pages(self) -> list[tuple[int, int]]: - """Get all (rank, page) pairs for this key.""" - return [(rank, page) for rank, page in self.rank_to_page.items()] - - def get_rank_page(self, rank: int) -> int | None: - """Get page index for a specific rank.""" - return self.rank_to_page.get(rank) - - def is_complete(self) -> bool: - """Check if all ranks in the TP world have allocated pages.""" - return len(self.rank_to_page) == self.tp_world_size - - -class GlobalMetadataState: - """Manages global metadata state across all ranks and keys.""" - - def __init__(self): - self.global_lock = threading.RLock() - self.rank_metadata: dict[int, RankFileMetadata] = {} - self.key_metadata: dict[str, KeyMetadata] = {} - - def clear(self) -> None: - """Clear all metadata state.""" - with self.global_lock: - self.rank_metadata.clear() - self.key_metadata.clear() - logger.info("Cleared all metadata state") - - def initialize_rank(self, rank: int, num_pages: int) -> None: - """Initialize a new rank with specified number of pages.""" - with self.global_lock: - if rank not in self.rank_metadata: - self.rank_metadata[rank] = RankFileMetadata( - rank, num_pages, list(range(num_pages)) - ) - logger.info("Initialized rank %s with %s pages", rank, num_pages) - - def allocate_pages_for_keys( - self, rank: int, keys: list[tuple[str, str]] - ) -> dict[str, int]: - """Allocate one page for each key on the specified rank. - - Args: - rank: Rank ID to allocate pages on - keys: List of keys to allocate pages for - - Returns: - Dictionary mapping key -> allocated page index - - """ - with self.global_lock: - if rank not in self.rank_metadata: - raise ValueError(f"Rank {rank} not initialized") - - # Batch allocate pages for all keys - num_pages_needed = len(keys) - allocated_pages = self.rank_metadata[rank].allocate_pages(num_pages_needed) - - if len(allocated_pages) < num_pages_needed: - logger.warning( - "Rank %s only allocated %s pages for %s keys", - rank, - len(allocated_pages), - num_pages_needed, - ) - - allocation_results = {} - for i, (key, prefix_key) in enumerate(keys): - if key in self.key_metadata: - key_meta = self.key_metadata[key] - if key_meta.is_complete() and rank in key_meta.rank_to_page: - # key is already fully written, reuse the existing page - # and release the allocated pages back to the free pool. - if i < len(allocated_pages): - self.rank_metadata[rank].release_pages([allocated_pages[i]]) - allocation_results[key] = key_meta.rank_to_page[rank] - continue - - if i < len(allocated_pages): - allocation_results[key] = allocated_pages[i] - else: - allocation_results[key] = -1 # No pages available - - return allocation_results - - def confirm_write_for_keys( - self, - rank: int, - key_confirmations: list[tuple[str, int]], - pages_to_release: list[int] | None = None, - ) -> None: - """Confirm write operations for keys and update metadata. - - Args: - rank: Rank ID that confirmed the writes - key_confirmations: List of (key, page_index) tuples - pages_to_release: List of page indices to release back to free pool - - """ - with self.global_lock: - # Confirm successful writes - for key, page_index in key_confirmations: - if key not in self.key_metadata: - # Need to determine tp_world_size from rank_metadata - tp_world_size = len(self.rank_metadata) - self.key_metadata[key] = KeyMetadata(key, {}, tp_world_size) - - # Add confirmed page to key metadata - self.key_metadata[key].add_rank_page(rank, page_index) - - # Release specified pages back to free pool - if pages_to_release: - self.rank_metadata[rank].release_pages(pages_to_release) - logger.debug( - "Released %s pages on rank %s: %s", - len(pages_to_release), - rank, - pages_to_release, - ) - - def batch_key_exists(self, keys: list[str]) -> list[bool]: - """Check if keys exist in metadata and all ranks have confirmed writes. - - Args: - keys: List of keys to check - - Returns: - List of boolean values indicating key existence and completion - - """ - with self.global_lock: - results = [] - for key in keys: - if key not in self.key_metadata: - results.append(False) - else: - # Check if all ranks in the TP world have confirmed writes - key_meta = self.key_metadata[key] - results.append(key_meta.is_complete()) - return results - - def get_key_locations(self, rank: int, keys: list[str]) -> list[int | None]: - """Get page indices for keys on a specific rank. - - Args: - rank: Rank ID to query - keys: List of keys to look up - - Returns: - List of page indices in the same order as input keys (None if key not found) - - """ - with self.global_lock: - if rank not in self.rank_metadata: - raise ValueError(f"Rank {rank} not initialized") - - results = [] - for key in keys: - if key in self.key_metadata: - key_meta = self.key_metadata[key] - if key_meta.is_complete(): - page_index = key_meta.get_rank_page(rank) - else: - page_index = None - - results.append(page_index) - else: - results.append(None) - - return results - - -class Hf3fsMetadataServer: - """HF3FS Metadata Server with improved key-based organization.""" - - def __init__(self, persistence_path: str | None = None, save_interval: int = 60): - self.state = GlobalMetadataState() - if HAS_ORJSON: - self.app = FastAPI(default_response_class=ORJSONResponse) - else: - self.app = FastAPI() - self._setup_routes() - - async def _read_json(self, request: Request) -> dict: - """Parse request JSON using orjson if available.""" - body = await request.body() - return orjson.loads(body) - - def _json_response(self, content: dict): - """Return ORJSONResponse when available to bypass jsonable_encoder.""" - if HAS_ORJSON: - return ORJSONResponse(content) - else: - return content - - def _setup_routes(self): - """Setup FastAPI routes for new API design.""" - self.app.post("/rank/{rank}/initialize")(self.initialize_rank) - self.app.post("/keys/batch_allocate")(self.batch_allocate_pages_for_keys) - self.app.post("/keys/confirm_write")(self.confirm_write_for_keys) - self.app.post("/keys/batch_exists")(self.batch_key_exists) - self.app.post("/keys/get_locations")(self.get_key_locations) - self.app.post("/clear")(self.clear) - - async def initialize_rank(self, rank: int, request: Request): - """Initialize a rank with specified number of pages.""" - data = await self._read_json(request) - role = data.get("role", "worker") - num_pages = data.get("num_pages", 0) - - if role == "scheduler": - return self._json_response( - {"message": "Scheduler role does not require initialization"} - ) - - if role == "worker" and num_pages > 0: - self.state.initialize_rank(rank, num_pages) - return self._json_response( - {"message": f"Rank {rank} initialized with {num_pages} pages"} - ) - else: - raise HTTPException( - status_code=400, detail="Invalid initialization parameters" - ) - - async def batch_allocate_pages_for_keys(self, request: Request): - """Allocate one page for each key on a specific rank.""" - data = await self._read_json(request) - rank = data.get("rank") - keys = data.get("keys", []) - - # Validate input format - if rank is None or not isinstance(keys, list): - raise HTTPException( - status_code=400, detail="Invalid request format: need 'rank' and 'keys'" - ) - - try: - # Perform allocation - results = self.state.allocate_pages_for_keys(rank, keys) - - # Convert results to response format - response = {"rank": rank, "results": list(results.items())} - return self._json_response(response) - except Exception as e: - raise HTTPException( - status_code=500, detail=f"Allocation failed: {str(e)}" - ) from e - - async def confirm_write_for_keys(self, request: Request): - """Confirm write operations for keys.""" - data = await self._read_json(request) - rank = data.get("rank") - confirmations = data.get("confirmations", []) - pages_to_release = data.get("pages_to_release", []) - - # Validate input format - if rank is None or not isinstance(confirmations, list): - raise HTTPException( - status_code=400, - detail="Invalid request format: need 'rank' and 'confirmations'", - ) - - try: - self.state.confirm_write_for_keys(rank, confirmations, pages_to_release) - - return Response(status_code=204) - - except Exception as e: - logger.error("Confirm write for keys failed: %s", e) - raise HTTPException( - status_code=500, detail=f"Confirmation failed: {str(e)}" - ) from e - - async def batch_key_exists(self, request: Request): - """Check if multiple keys exist in metadata.""" - data = await self._read_json(request) - keys = data.get("keys", []) - - if not isinstance(keys, list): - raise HTTPException(status_code=400, detail="Invalid keys format") - - try: - exists_results = self.state.batch_key_exists(keys) - return self._json_response({"exists": exists_results}) - except Exception as e: - raise HTTPException( - status_code=500, detail=f"Key existence check failed: {str(e)}" - ) from e - - async def get_key_locations(self, request: Request): - """Get page indices for keys on a specific rank.""" - data = await self._read_json(request) - rank = data.get("rank") - keys = data.get("keys", []) - - # Validate input format - if rank is None or not isinstance(keys, list): - raise HTTPException( - status_code=400, detail="Invalid request format: need 'rank' and 'keys'" - ) - - try: - # Get key locations - locations = self.state.get_key_locations(rank, keys) - return self._json_response({"locations": locations}) - except Exception as e: - raise HTTPException( - status_code=500, detail=f"Failed to get key locations: {str(e)}" - ) from e - - async def clear(self, request: Request): - """Clear the metadata server.""" - self.state.clear() - return Response(status_code=204) - - def run(self, host: str = "0.0.0.0", port: int = 18000): - """Run the metadata server.""" - import uvicorn - - logger.info("Starting improved metadata server on http://%s:%s", host, port) - uvicorn.run(self.app, host=host, port=port) - - -# --- Client implementation --- -class Hf3fsMetadataInterface(ABC): - """Interface for HF3FS metadata operations.""" - - @abstractmethod - def initialize(self, rank: int, num_pages: int = 0, role: str = "worker") -> None: - """Initialize the metadata service with specified number of pages.""" - pass - - @abstractmethod - def allocate_pages_for_keys( - self, rank: int, keys: list[tuple[str, str]] - ) -> list[tuple[str, int]]: - """Allocate one page for each key on the specified rank.""" - pass - - @abstractmethod - def confirm_write_for_keys( - self, - rank: int, - key_confirmations: list[tuple[str, int]], - pages_to_release: list[int] | None = None, - ) -> None: - """Confirm write operations for keys and optionally release pages.""" - pass - - @abstractmethod - def batch_key_exists(self, keys: list[str]) -> list[bool]: - """Check if keys exist and are complete across all ranks.""" - pass - - @abstractmethod - def get_key_locations(self, rank: int, keys: list[str]) -> list[int]: - """Get page indices for keys on a specific rank.""" - pass - - -class Hf3fsGlobalMetadataClient(Hf3fsMetadataInterface): - """Global HTTP metadata client for HF3FS.""" - - def __init__(self, base_url: str = "http://localhost:18000", max_retries: int = 3): - self.base_url = base_url.rstrip("/") - self._session = requests.Session() - - retry_strategy = Retry( - total=max_retries, - backoff_factor=0.3, - status_forcelist=[500, 502, 503, 504], - allowed_methods=["GET", "POST"], - ) - adapter = HTTPAdapter(max_retries=retry_strategy) - self._session.mount("http://", adapter) - - def _post(self, endpoint: str, json_data: dict) -> dict: - """Make POST request to metadata server.""" - try: - url = f"{self.base_url}/{endpoint}" - headers = {"Content-Type": "application/json"} - if HAS_ORJSON: - payload = orjson.dumps(json_data) - else: - import json - - payload = json.dumps(json_data).encode("utf-8") - response = self._session.post(url, data=payload, headers=headers) - response.raise_for_status() - - if response.status_code == 204 or not response.content: - return {} - if HAS_ORJSON: - return orjson.loads(response.content) - else: - return response.json() - except requests.exceptions.RequestException as e: - logger.error("Failed to POST to %s after retries: %s", endpoint, e) - raise RuntimeError(f"Failed to connect to metadata server: {e}") from e - - def initialize(self, rank: int, num_pages: int = 0, role: str = "worker") -> None: - """Initialize a rank with specified number of pages.""" - self._post(f"rank/{rank}/initialize", {"num_pages": num_pages, "role": role}) - - def allocate_pages_for_keys( - self, rank: int, keys: list[tuple[str, str]] - ) -> list[tuple[str, int]]: - """Allocate pages for keys on the specified rank.""" - response = self._post("keys/batch_allocate", {"rank": rank, "keys": keys}) - - # Convert response to expected format - return response.get("results", {}) - - def confirm_write_for_keys( - self, - rank: int, - key_confirmations: list[tuple[str, int]], - pages_to_release: list[int] | None = None, - ) -> None: - """Confirm write operations for keys and optionally release pages.""" - payload = { - "rank": rank, - "confirmations": key_confirmations, - "pages_to_release": pages_to_release or [], - } - - self._post("keys/confirm_write", payload) - - def batch_key_exists(self, keys: list[str]) -> list[bool]: - """Check if keys exist and are complete across all ranks.""" - response = self._post("keys/batch_exists", {"keys": keys}) - return response.get("exists", []) - - def get_key_locations(self, rank: int, keys: list[str]) -> list[int]: - """Get page indices for keys on a specific rank.""" - response = self._post("keys/get_locations", {"rank": rank, "keys": keys}) - return response.get("locations", []) - - -def run_metadata_server( - host: str = "0.0.0.0", - port: int = 18000, -): - """Run the improved HF3FS metadata server.""" - server = Hf3fsMetadataServer() - server.run(host=host, port=port) - - -# --- Main Execution --- -if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Improved HF3FS Metadata Server") - parser.add_argument( - "--host", type=str, default="0.0.0.0", help="Host to bind the server to." - ) - parser.add_argument( - "--port", type=int, default=18000, help="Port to run the server on." - ) - args = parser.parse_args() - - run_metadata_server(args.host, args.port) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/hf3fs/utils/__init__.py b/vllm/distributed/kv_transfer/kv_connector/v1/hf3fs/utils/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/hf3fs/utils/common.py b/vllm/distributed/kv_transfer/kv_connector/v1/hf3fs/utils/common.py deleted file mode 100644 index b47de73c99..0000000000 --- a/vllm/distributed/kv_transfer/kv_connector/v1/hf3fs/utils/common.py +++ /dev/null @@ -1,139 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -import threading -from dataclasses import dataclass, field -from typing import Optional - -from vllm.distributed.kv_transfer.kv_connector.v1.base import KVConnectorMetadata -from vllm.v1.request import Request - - -class AtomicCounter: - """Thread-safe atomic counter for round-robin operations.""" - - def __init__(self, n: int): - assert n > 0, "Counter size must be positive" - self._n = n - self._value = 0 - self._lock = threading.Lock() - - def next(self) -> int: - """Get next value in round-robin fashion.""" - with self._lock: - current = self._value - self._value = (current + 1) % self._n - return current - - -@dataclass -class LoadBlockInfo: - """Operation for loading blocks from external storage.""" - - num_computed_blocks: int - num_blocks_to_load: int - need_fetch_block_ids: list[int] - - -@dataclass -class SaveBlockInfo: - """Operation for saving blocks to external storage.""" - - skip_leading_blocks: int - - -@dataclass -class RequestSchedulingState: - """Unified request scheduling state management.""" - - request_id: str - request: Request | None = None - - # Token and block tracking - token_ids: list[int] = field(default_factory=list) - allocated_block_ids: list[int] = field(default_factory=list) - num_saved_blocks: int = 0 - - # Load operation info - load_op: LoadBlockInfo | None = None - - # Scheduling phase - phase: str = "NEW" # NEW -> WAITING_TO_LOAD -> ACTIVE -> FINISHED - - def needs_loading(self) -> bool: - """Check if request needs loading.""" - return self.load_op is not None and self.load_op.num_blocks_to_load > 0 - - def is_ready_to_load(self) -> bool: - """Check if request is ready for loading.""" - return self.phase == "WAITING_TO_LOAD" and self.needs_loading() - - def update_tokens_and_blocks(self, new_token_ids: list[int], new_block_ids) -> None: - """Update with new tokens and blocks.""" - if new_token_ids: - self.token_ids.extend(new_token_ids) - - if new_block_ids is not None: - normalized_block_ids = self._normalize_block_ids(new_block_ids) - self.allocated_block_ids.extend(normalized_block_ids) - - def _normalize_block_ids(self, block_ids) -> list[int]: - """Normalize block_ids to list format.""" - if not block_ids: - return [] - if isinstance(block_ids, tuple): - return block_ids[0] if block_ids else [] - if isinstance(block_ids, list): - return block_ids - return [] - - -@dataclass -class HF3FSRequestMetadata: - """Metadata for a single request in HF3FS connector.""" - - request_id: str - token_ids: list[int] - block_ids: list[int] - load_block_op: LoadBlockInfo | None = None - save_block_op: SaveBlockInfo | None = None - - @staticmethod - def from_scheduling_state( - state: "RequestSchedulingState", - block_size: int, - load_op: LoadBlockInfo | None = None, - skip_leading_blocks: int | None = None, - ) -> Optional["HF3FSRequestMetadata"]: - """Create request metadata from scheduling state.""" - token_count = len(state.token_ids) - total_blocks = token_count // block_size - - skip_blocks = ( - state.num_saved_blocks - if skip_leading_blocks is None - else skip_leading_blocks - ) - - new_blocks_to_save = total_blocks - state.num_saved_blocks - if new_blocks_to_save <= 0 and load_op is None: - return None - - state.num_saved_blocks = total_blocks - return HF3FSRequestMetadata( - request_id=state.request_id, - token_ids=state.token_ids, - block_ids=state.allocated_block_ids, - load_block_op=load_op, - save_block_op=SaveBlockInfo(skip_leading_blocks=skip_blocks), - ) - - -class HF3FSConnectorMetadata(KVConnectorMetadata): - """Container for HF3FS connector metadata.""" - - def __init__(self): - self.requests: list[HF3FSRequestMetadata] = [] - - def add_request(self, request_metadata: HF3FSRequestMetadata) -> None: - """Add request to metadata.""" - self.requests.append(request_metadata) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/hf3fs/utils/gather_scatter_helper.py b/vllm/distributed/kv_transfer/kv_connector/v1/hf3fs/utils/gather_scatter_helper.py deleted file mode 100644 index 55c3c8407d..0000000000 --- a/vllm/distributed/kv_transfer/kv_connector/v1/hf3fs/utils/gather_scatter_helper.py +++ /dev/null @@ -1,262 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -import torch - -from vllm.logger import init_logger -from vllm.triton_utils import tl, triton -from vllm.utils.torch_utils import async_tensor_h2d - - -@triton.jit -def kv_cache_scatter_kernel( - kv_cache_ptrs_ptr, - source_ptr, - token_indices_ptr, - num_tokens_in_block, - total_token_in_kvcache, - num_layers, - tokens_per_block, - block_stride, - head_stride, - token_stride, - content_stride, - num_heads: tl.constexpr, - content_size: tl.constexpr, - BLOCK_SIZE: tl.constexpr, -): - layer_idx = tl.program_id(0) - token_pos = tl.program_id(1) - - if layer_idx >= num_layers or token_pos >= num_tokens_in_block: - return - - token_idx = tl.load(token_indices_ptr + token_pos) - kv_cache_ptr = tl.cast(tl.load(kv_cache_ptrs_ptr + layer_idx), source_ptr.dtype) - - if token_idx >= total_token_in_kvcache: - return - - block_idx = token_idx // tokens_per_block - state_idx = token_idx % tokens_per_block - hidden_size = num_heads * content_size - for i in range(0, hidden_size, BLOCK_SIZE): - offset = i + tl.arange(0, BLOCK_SIZE) - mask = offset < hidden_size - head_idx = offset // content_size - content_idx = offset % content_size - source_offset = ( - (layer_idx * num_heads + head_idx) * num_tokens_in_block + token_pos - ) * content_size + content_idx - target_offset = ( - block_idx * block_stride - + head_idx * head_stride - + state_idx * token_stride - + content_idx * content_stride - ) - value = tl.load(source_ptr + source_offset, mask=mask) - tl.store(kv_cache_ptr + target_offset, value, mask=mask) - - -@triton.jit -def kv_cache_gather_kernel( - kv_cache_ptrs_ptr, - dst_ptr, - token_indices_ptr, - num_tokens_in_block, - total_token_in_kvcache, - num_layers, - tokens_per_block, - block_stride, - head_stride, - token_stride, - content_stride, - num_heads: tl.constexpr, - content_size: tl.constexpr, - BLOCK_SIZE: tl.constexpr, -): - layer_idx = tl.program_id(0) - token_pos = tl.program_id(1) - - if layer_idx >= num_layers or token_pos >= num_tokens_in_block: - return - - token_idx = tl.load(token_indices_ptr + token_pos) - kv_cache_ptr = tl.cast(tl.load(kv_cache_ptrs_ptr + layer_idx), dst_ptr.dtype) - - if token_idx >= total_token_in_kvcache: - return - - block_idx = token_idx // tokens_per_block - state_idx = token_idx % tokens_per_block - hidden_size = num_heads * content_size - for i in range(0, hidden_size, BLOCK_SIZE): - offset = i + tl.arange(0, BLOCK_SIZE) - mask = offset < hidden_size - head_idx = offset // content_size - content_idx = offset % content_size - kvcache_offset = ( - block_idx * block_stride - + head_idx * head_stride - + state_idx * token_stride - + content_idx * content_stride - ) - dst_offset = ( - (layer_idx * num_heads + head_idx) * num_tokens_in_block + token_pos - ) * content_size + content_idx - value = tl.load(kv_cache_ptr + kvcache_offset, mask=mask) - tl.store(dst_ptr + dst_offset, value, mask=mask) - - -def scatter_kv_caches( - kv_caches_ptrs: torch.Tensor, - total_token_in_kvcache: int, - src_tensor: torch.Tensor, - token_indices: list[int], - tokens_per_block: int, - num_heads: int, - content_size: int, - kv_cache_strides: tuple[int, ...], -) -> None: - """Scatter KV cache data from source tensor to KV cache storage. - - Args: - kv_caches_ptrs: Tensor of KV cache pointers (one per layer) - total_token_in_kvcache: Total number of tokens in KV cache - src_tensor: Source ``[L, H, N, C]`` tensor containing data to scatter - token_indices: List of token positions to update - tokens_per_block: Number of stored states in each cache block - num_heads: Size of the H axis - content_size: Size of the C axis - kv_cache_strides: Element strides of each ``[B, H, N, C]`` layer view - - """ - num_layers = len(kv_caches_ptrs) - num_tokens_in_block = len(token_indices) - - assert src_tensor.shape == ( - num_layers, - num_heads, - num_tokens_in_block, - content_size, - ) - assert len(kv_cache_strides) == 4 - - device = src_tensor.device - token_indices_tensor = async_tensor_h2d( - token_indices, device=device, dtype=torch.int32 - ) - - grid = (num_layers, num_tokens_in_block) - BLOCK_SIZE = 128 - - kv_cache_scatter_kernel[grid]( - kv_caches_ptrs, - src_tensor, - token_indices_tensor, - num_tokens_in_block, - total_token_in_kvcache, - num_layers, - tokens_per_block, - *kv_cache_strides, - num_heads=num_heads, - content_size=content_size, - BLOCK_SIZE=BLOCK_SIZE, - ) - - -def gather_kv_caches( - kv_caches_ptrs: torch.Tensor, - total_token_in_kvcache: int, - dst_tensor: torch.Tensor, - token_indices: list[int], - tokens_per_block: int, - num_heads: int, - content_size: int, - kv_cache_strides: tuple[int, ...], -) -> None: - """Gather KV cache data from KV cache storage to destination tensor. - - Args: - kv_caches_ptrs: Tensor of KV cache pointers (one per layer) - total_token_in_kvcache: Total number of tokens in KV cache - dst_tensor: Destination ``[L, H, N, C]`` tensor - token_indices: List of token positions to gather - tokens_per_block: Number of stored states in each cache block - num_heads: Size of the H axis - content_size: Size of the C axis - kv_cache_strides: Element strides of each ``[B, H, N, C]`` layer view - - """ - num_layers = kv_caches_ptrs.shape[0] - num_tokens_in_block = len(token_indices) - - assert dst_tensor.shape == ( - num_layers, - num_heads, - num_tokens_in_block, - content_size, - ) - assert len(kv_cache_strides) == 4 - - device = dst_tensor.device - token_indices_tensor = async_tensor_h2d( - token_indices, device=device, dtype=torch.int32 - ) - - grid = (num_layers, num_tokens_in_block) - BLOCK_SIZE = 128 - - kv_cache_gather_kernel[grid]( - kv_caches_ptrs, - dst_tensor, - token_indices_tensor, - num_tokens_in_block, - total_token_in_kvcache, - num_layers, - tokens_per_block, - *kv_cache_strides, - num_heads=num_heads, - content_size=content_size, - BLOCK_SIZE=BLOCK_SIZE, - ) - - -class CopyBufferAllocator: - """Memory pool for tensor buffers to avoid frequent allocation/deallocation.""" - - def __init__( - self, device: torch.device, dtype: torch.dtype, shape: list, max_count: int - ): - self._shape = shape - self._max_count = max_count - self._device = device - self._free_buffers = [ - torch.empty(shape, dtype=dtype, device=device) for _ in range(max_count) - ] - self._inuse_count = 0 - - def alloc_buffer(self, count: int) -> list[torch.Tensor] | None: - """Allocate buffers from the pool.""" - if count == 0: - return [] - - if self._inuse_count + count <= self._max_count: - self._inuse_count += count - result = self._free_buffers[-count:] - del self._free_buffers[-count:] - return result - return None - - def free_buffer(self, buffers: list[torch.Tensor]) -> None: - """Return buffers to the pool.""" - if not buffers: - return - - if self._inuse_count >= len(buffers): - self._inuse_count -= len(buffers) - self._free_buffers.extend(buffers) - else: - raise RuntimeError("Attempted to free more buffers than allocated") - - -logger = init_logger(__name__) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/hf3fs/utils/hf3fs_mock_client.py b/vllm/distributed/kv_transfer/kv_connector/v1/hf3fs/utils/hf3fs_mock_client.py deleted file mode 100644 index 3914663a62..0000000000 --- a/vllm/distributed/kv_transfer/kv_connector/v1/hf3fs/utils/hf3fs_mock_client.py +++ /dev/null @@ -1,133 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -import logging -import os - -import torch - -logger = logging.getLogger(__name__) -HF3FS_AVAILABLE = True - - -class Hf3fsClient: - """Mock HF3FS client using file backend for debugging and testing.""" - - def __init__(self, path: str, size: int, bytes_per_page: int, entries: int): - self._size = size - self._bytes_per_page = bytes_per_page - self._entries = entries - self._file_path = path - - self._ensure_file_exists() - logger.debug("Initialized mock HF3FS client: %s (%d bytes)", path, size) - - def _ensure_file_exists(self) -> None: - """Create file if it doesn't exist.""" - if not os.path.exists(self._file_path): - with open(self._file_path, "w+b") as f: - f.truncate(self._size) - - def batch_read(self, offsets: list[int], tensors: list[torch.Tensor]) -> list[int]: - """Read data from file at specified offsets into tensors.""" - results = [] - - try: - with open(self._file_path, "rb") as f: - for offset, tensor in zip(offsets, tensors): - num_bytes = tensor.numel() * tensor.element_size() - - if offset < 0 or offset + num_bytes > self._size: - results.append(-1) - continue - - f.seek(offset) - buffer_data = f.read(num_bytes) - - if len(buffer_data) == num_bytes == self._bytes_per_page: - tensor_data = self._convert_buffer_to_tensor( - buffer_data, tensor.dtype - ) - tensor.copy_( - tensor_data.reshape(tensor.shape).to(tensor.device) - ) - results.append(self._bytes_per_page) - else: - logger.error( - "Read size mismatch: got %d, expected %d", - len(buffer_data), - num_bytes, - ) - results.append(-1) - except Exception as e: - logger.error("Batch read error: %s", e) - results.extend([-1] * (len(offsets) - len(results))) - - return results - - def _convert_buffer_to_tensor( - self, buffer_data: bytes, dtype: torch.dtype - ) -> torch.Tensor: - """Convert buffer data to tensor with proper dtype handling.""" - if dtype == torch.bfloat16: - tensor_data = torch.frombuffer(buffer_data, dtype=torch.uint16) - return tensor_data.view(dtype=torch.bfloat16) - else: - return torch.frombuffer(buffer_data, dtype=dtype) - - def batch_write( - self, offsets: list[int], tensors: list[torch.Tensor], event: torch.cuda.Event - ) -> list[int]: - """Write data from tensors to file at specified offsets.""" - results = [] - - try: - torch.cuda.current_stream().wait_event(event) - - # Convert tensors to bytes - data_bytes_list = [self._tensor_to_bytes(tensor) for tensor in tensors] - - # Write to file - with open(self._file_path, "r+b") as f: - for offset, data_bytes in zip(offsets, data_bytes_list): - if offset < 0 or offset + len(data_bytes) > self._size: - results.append(-1) - continue - - f.seek(offset) - bytes_written = f.write(data_bytes) - - if bytes_written == len(data_bytes) == self._bytes_per_page: - results.append(self._bytes_per_page) - else: - logger.error( - "Write size mismatch: wrote %d, expected %d", - bytes_written, - self._bytes_per_page, - ) - results.append(-1) - - except Exception as e: - logger.error("Batch write error: %s", e) - results.extend([-1] * (len(offsets) - len(results))) - - return results - - def _tensor_to_bytes(self, tensor: torch.Tensor) -> bytes: - """Convert tensor to bytes with proper dtype handling.""" - cpu_tensor = tensor.cpu() - if cpu_tensor.dtype == torch.bfloat16: - return cpu_tensor.view(dtype=torch.uint16).numpy().tobytes() - else: - return cpu_tensor.numpy().tobytes() - - def get_size(self) -> int: - """Get the total size of the storage file.""" - return self._size - - def close(self) -> None: - """Close the client (no-op for file backend).""" - pass - - def flush(self) -> None: - """Flush any pending writes (no-op for file backend).""" - pass diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/hf3fs/utils/hf3fs_utils.cpp b/vllm/distributed/kv_transfer/kv_connector/v1/hf3fs/utils/hf3fs_utils.cpp deleted file mode 100644 index 9dbeb251d0..0000000000 --- a/vllm/distributed/kv_transfer/kv_connector/v1/hf3fs/utils/hf3fs_utils.cpp +++ /dev/null @@ -1,57 +0,0 @@ -#include -#include -#include -#include - -void read_shm(const torch::Tensor& shm, const torch::Tensor& pin, - std::vector dst, uint64_t stream_ptr) { - py::gil_scoped_release release; - - cudaStream_t stream = reinterpret_cast(stream_ptr); - - // Copy from shared memory to pinned memory - char* shm_ptr = static_cast(shm.data_ptr()); - char* src_ptr = static_cast(pin.data_ptr()); - std::memcpy(src_ptr, shm_ptr, shm.numel() * shm.element_size()); - - // Copy from pinned memory to GPU tensors - size_t current = 0; - for (size_t i = 0; i < dst.size(); ++i) { - auto& t = dst[i]; - size_t t_bytes = t.numel() * t.element_size(); - char* dst_ptr = static_cast(t.data_ptr()); - cudaMemcpyAsync(dst_ptr, src_ptr + current, t_bytes, cudaMemcpyHostToDevice, - stream); - current += t_bytes; - } - cudaStreamSynchronize(stream); -} - -void write_shm(const std::vector src, torch::Tensor& shm, - const torch::Tensor& pin, uint64_t stream_ptr) { - py::gil_scoped_release release; - - cudaStream_t stream = reinterpret_cast(stream_ptr); - - // Copy from GPU tensors to pinned memory - char* dst_ptr = static_cast(pin.data_ptr()); - size_t current = 0; - for (size_t i = 0; i < src.size(); ++i) { - auto& t = src[i]; - size_t t_bytes = t.numel() * t.element_size(); - char* src_ptr = static_cast(t.data_ptr()); - cudaMemcpyAsync(dst_ptr + current, src_ptr, t_bytes, cudaMemcpyDeviceToHost, - stream); - current += t_bytes; - } - cudaStreamSynchronize(stream); - - // Copy from pinned memory to shared memory - char* shm_ptr = static_cast(shm.data_ptr()); - std::memcpy(shm_ptr, dst_ptr, shm.numel() * shm.element_size()); -} - -PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { - m.def("read_shm", &read_shm, "Read tensors from shared memory"); - m.def("write_shm", &write_shm, "Write tensors to shared memory"); -} \ No newline at end of file diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/hisparse/__init__.py b/vllm/distributed/kv_transfer/kv_connector/v1/hisparse/__init__.py deleted file mode 100644 index 208f01a7cb..0000000000 --- a/vllm/distributed/kv_transfer/kv_connector/v1/hisparse/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/hisparse/connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/hisparse/connector.py deleted file mode 100644 index 4f17f1e737..0000000000 --- a/vllm/distributed/kv_transfer/kv_connector/v1/hisparse/connector.py +++ /dev/null @@ -1,408 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Connector boundary for HiSparse scheduler and worker state.""" - -from __future__ import annotations - -from dataclasses import dataclass -from typing import TYPE_CHECKING, Any - -import torch - -from vllm.config import VllmConfig -from vllm.distributed.kv_transfer.kv_connector.v1.base import ( - KVConnectorBase_V1, - KVConnectorMetadata, - KVConnectorRole, - KVConnectorWorkerMetadata, - SupportsHMA, -) -from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.stats import ( - HiSparseKVConnectorStats, - HiSparsePromMetrics, -) -from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.worker import ( - HiSparseConnectorWorker, -) -from vllm.distributed.kv_transfer.kv_connector.v1.metrics import ( - KVConnectorPromMetrics, - KVConnectorStats, - PromMetric, - PromMetricT, -) -from vllm.logger import init_logger -from vllm.v1.attention.backend import AttentionMetadata -from vllm.v1.core.kv_cache_utils import KVCacheBlockCopy -from vllm.v1.core.sched.output import SchedulerOutput -from vllm.v1.hisparse.coordinator import get_hisparse_coordinator -from vllm.v1.hisparse.layout import get_hisparse_steady_state_concurrency -from vllm.v1.hisparse.types import ( - SparseKVOffloadCommand, - SparseKVResidencyUpdate, - SparseKVRowMirror, -) -from vllm.v1.outputs import KVConnectorOutput - -if TYPE_CHECKING: - from vllm.forward_context import ForwardContext - from vllm.v1.core.kv_cache_manager import KVCacheBlocks, KVCacheManager - from vllm.v1.hisparse.coordinator import HiSparseCoordinator - from vllm.v1.kv_cache_interface import KVCacheConfig - from vllm.v1.request import Request - -logger = init_logger(__name__) - - -@dataclass -class HiSparseConnectorMetadata(KVConnectorMetadata): - command: SparseKVOffloadCommand | None - host_block_copies: tuple[KVCacheBlockCopy, ...] - source_block_ids: tuple[int, ...] - row_mirrors: dict[str, tuple[SparseKVRowMirror, ...]] - all_context_pages_resident: bool - residency_updates: dict[str, SparseKVResidencyUpdate] - - -@dataclass -class HiSparseConnectorWorkerMetadata(KVConnectorWorkerMetadata): - enqueued_transfer_counts: dict[int, int] - completed_transfer_counts: dict[int, int] - completed_host_copy_dst_ids: tuple[int, ...] = () - - def aggregate(self, other: KVConnectorWorkerMetadata) -> KVConnectorWorkerMetadata: - assert isinstance(other, HiSparseConnectorWorkerMetadata) - - def add_counts(first: dict[int, int], second: dict[int, int]) -> dict[int, int]: - combined = first.copy() - for transfer_id, count in second.items(): - combined[transfer_id] = combined.get(transfer_id, 0) + count - return combined - - return HiSparseConnectorWorkerMetadata( - enqueued_transfer_counts=add_counts( - self.enqueued_transfer_counts, other.enqueued_transfer_counts - ), - completed_transfer_counts=add_counts( - self.completed_transfer_counts, other.completed_transfer_counts - ), - # Every worker runs the same copies, so a union is the ack. - completed_host_copy_dst_ids=tuple( - {*self.completed_host_copy_dst_ids, *other.completed_host_copy_dst_ids} - ), - ) - - -class HiSparseConnectorScheduler: - def __init__( - self, - *, - async_speculative: bool, - draft_kv_lookahead: int = 0, - ) -> None: - self.coordinator: HiSparseCoordinator | None = None - self.async_speculative = async_speculative - self.draft_kv_lookahead = draft_kv_lookahead - self.requests: dict[str, Request] = {} - - def bind_coordinator(self, coordinator: HiSparseCoordinator) -> None: - assert self.coordinator is None - self.coordinator = coordinator - - def get_kv_connector_stats(self) -> HiSparseKVConnectorStats | None: - if self.coordinator is None: - return None - host_pool = self.coordinator.get_host_block_pool() - assert host_pool is not None - stats = HiSparseKVConnectorStats() - stats.record_host_usage( - host_pool.get_usage(), len(self.coordinator.pending_spills) - ) - if host_pool.metrics_collector is not None: - stats.record_host_evictions(host_pool.metrics_collector.drain_events()) - return stats - - def build_connector_meta( - self, scheduler_output: SchedulerOutput - ) -> HiSparseConnectorMetadata: - assert self.coordinator is not None - # HiSparse rebinds worker state every step, so the load must start - # before the forward rather than being deferred to post-forward. - scheduler_output.has_sync_kv_loads = True - num_computed_tokens = { - request.req_id: request.num_computed_tokens - for request in scheduler_output.scheduled_new_reqs - } - num_computed_tokens.update( - zip( - scheduler_output.scheduled_cached_reqs.req_ids, - scheduler_output.scheduled_cached_reqs.num_computed_tokens, - ) - ) - scheduled_requests = tuple( - ( - request_id, - num_computed_tokens[request_id], - scheduled_count, - ) - for request_id, scheduled_count in ( - scheduler_output.num_scheduled_tokens.items() - ) - ) - # Runs first: it plans this step's transfers and residency changes. - self.coordinator.advance_scheduled( - ( - request_id, - min(start + count, self.requests[request_id].num_tokens), - ) - for request_id, start, count in scheduled_requests - ) - command = self.coordinator.build_offload_command() - host_block_copies = self.coordinator.take_host_block_copies() - source_group_id = self.coordinator.host_group_id - assert source_group_id is not None - source_block_ids = [ - block_id - for request in scheduler_output.scheduled_new_reqs - for block_id in request.block_ids[source_group_id] - ] - for new_block_ids in scheduler_output.scheduled_cached_reqs.new_block_ids: - if new_block_ids is not None: - source_block_ids.extend(new_block_ids[source_group_id]) - row_mirrors = {} - for request_id, scheduled_start, scheduled_count in scheduled_requests: - mirror_start = scheduled_start - if self.async_speculative: - mirror_start = max( - 0, - scheduled_start - self.requests[request_id].num_output_placeholders, - ) - row_mirrors[request_id] = self.coordinator.build_row_mirrors( - ( - ( - request_id, - mirror_start, - scheduled_count - + scheduled_start - - mirror_start - + self.draft_kv_lookahead, - ), - ), - ) - return HiSparseConnectorMetadata( - command, - host_block_copies, - tuple(source_block_ids), - row_mirrors, - self.coordinator.all_context_pages_resident(scheduled_requests), - self.coordinator.take_residency_updates( - scheduler_output.num_scheduled_tokens - ), - ) - - def update_connector_output(self, connector_output: KVConnectorOutput) -> None: - assert self.coordinator is not None - for request_id in connector_output.finished_recving or (): - self.coordinator.finish_host_import( - request_id, failed=request_id in connector_output.failed_recving - ) - metadata = connector_output.kv_connector_worker_meta - if metadata is None: - return - assert isinstance(metadata, HiSparseConnectorWorkerMetadata) - self.coordinator.release_completed_host_copies( - metadata.completed_host_copy_dst_ids - ) - self.coordinator.update_spills( - metadata.enqueued_transfer_counts, - metadata.completed_transfer_counts, - ) - - -class HiSparseConnector(KVConnectorBase_V1, SupportsHMA): - """Join the scheduler coordinator to the worker's transfer engine.""" - - @classmethod - def get_required_kvcache_layout(cls, vllm_config: VllmConfig) -> str: - return "BLHNC" - - def __init__( - self, - vllm_config: VllmConfig, - role: KVConnectorRole, - kv_cache_config: KVCacheConfig, - ) -> None: - super().__init__(vllm_config, role, kv_cache_config) - if kv_cache_config.hisparse_host_num_blocks is None: - raise ValueError("HiSparseConnector requires a HiSparse host pool") - self.connector_scheduler: HiSparseConnectorScheduler | None = None - self.connector_worker: HiSparseConnectorWorker | None = None - if role == KVConnectorRole.SCHEDULER: - speculative_config = vllm_config.speculative_config - self.connector_scheduler = HiSparseConnectorScheduler( - async_speculative=bool( - vllm_config.scheduler_config.async_scheduling - and speculative_config is not None - ), - draft_kv_lookahead=vllm_config.num_lookahead_tokens, - ) - max_model_len = vllm_config.model_config.max_model_len - steady_concurrency = get_hisparse_steady_state_concurrency( - vllm_config, kv_cache_config - ) - logger.info_once( - "HiSparse steady-state maximum concurrency for %s tokens per " - "request: %.2fx (running requests reading from host).", - f"{max_model_len:,}", - steady_concurrency, - ) - elif role == KVConnectorRole.WORKER: - self.connector_worker = HiSparseConnectorWorker( - vllm_config, kv_cache_config - ) - else: - raise ValueError(f"Unsupported KV connector role: {role}") - - def bind_kv_cache_manager(self, kv_cache_manager: KVCacheManager) -> None: - assert self.connector_scheduler is not None - self.connector_scheduler.bind_coordinator( - get_hisparse_coordinator(kv_cache_manager) - ) - - @property - def requires_kv_delivery(self) -> bool: - return False - - def has_pending_push_work(self) -> bool: - assert self.connector_scheduler is not None - assert self.connector_scheduler.coordinator is not None - return self.connector_scheduler.coordinator.has_pending_work() - - def has_pending_block_frees(self) -> bool: - assert self.connector_scheduler is not None - assert self.connector_scheduler.coordinator is not None - return self.connector_scheduler.coordinator.has_pending_reclamation() - - def finish_forward(self) -> None: - assert self.connector_worker is not None - self.connector_worker.finish_forward() - - def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]) -> None: - assert self.connector_worker is not None - self.connector_worker.register_kv_caches(kv_caches) - - def reset_capture_state(self) -> None: - assert self.connector_worker is not None - self.connector_worker.reset_hot_state() - - def get_kv_connector_stats(self) -> KVConnectorStats | None: - if self.connector_worker is not None: - return self.connector_worker.get_kv_connector_stats() - if self.connector_scheduler is not None: - return self.connector_scheduler.get_kv_connector_stats() - return None - - @classmethod - def build_kv_connector_stats( - cls, data: dict[str, Any] | None = None - ) -> KVConnectorStats | None: - return ( - HiSparseKVConnectorStats(data=data) - if data is not None - else HiSparseKVConnectorStats() - ) - - @classmethod - def build_prom_metrics( - cls, - vllm_config: VllmConfig, - metric_types: dict[type[PromMetric], type[PromMetricT]], - labelnames: list[str], - per_engine_labelvalues: dict[int, list[object]], - ) -> KVConnectorPromMetrics: - return HiSparsePromMetrics( - vllm_config, metric_types, labelnames, per_engine_labelvalues - ) - - def start_load_kv(self, forward_context: ForwardContext, **kwargs: Any) -> None: - assert self.connector_worker is not None - metadata = self._get_connector_metadata() - assert isinstance(metadata, HiSparseConnectorMetadata) - request_state_indices = kwargs.get("request_state_indices") - assert request_state_indices is None or isinstance( - request_state_indices, torch.Tensor - ) - request_ids = kwargs.get("request_ids") - assert request_ids is None or isinstance(request_ids, list) - attn_metadata = kwargs.get("attn_metadata") - assert attn_metadata is None or isinstance(attn_metadata, dict) - self.connector_worker.start_step( - metadata, - request_state_indices, - request_ids, - num_tokens=int(kwargs.get("num_tokens") or 0), - ) - self.connector_worker.prepare_forward(attn_metadata) - - def wait_for_layer_load(self, layer_name: str) -> None: - return - - def save_kv_layer( - self, - layer_name: str, - kv_layer: torch.Tensor, - attn_metadata: AttentionMetadata, - **kwargs: Any, - ) -> None: - return - - def wait_for_save(self) -> None: - return - - def build_connector_worker_meta(self) -> KVConnectorWorkerMetadata | None: - assert self.connector_worker is not None - enqueued, completed = self.connector_worker.take_transfer_updates() - host_copies = self.connector_worker.take_completed_host_copies() - if not enqueued and not completed and not host_copies: - return None - return HiSparseConnectorWorkerMetadata( - enqueued_transfer_counts={transfer_id: 1 for transfer_id in enqueued}, - completed_transfer_counts={transfer_id: 1 for transfer_id in completed}, - completed_host_copy_dst_ids=tuple(host_copies), - ) - - def shutdown(self) -> None: - if self.connector_worker is not None: - self.connector_worker.shutdown() - - def get_num_new_matched_tokens( - self, request: Request, num_computed_tokens: int - ) -> tuple[int | None, bool]: - return 0, False - - def update_state_after_alloc( - self, - request: Request, - blocks: KVCacheBlocks, - num_external_tokens: int, - ) -> None: - assert self.connector_scheduler is not None - self.connector_scheduler.requests[request.request_id] = request - - def build_connector_meta( - self, scheduler_output: SchedulerOutput - ) -> KVConnectorMetadata: - assert self.connector_scheduler is not None - return self.connector_scheduler.build_connector_meta(scheduler_output) - - def update_connector_output(self, connector_output: KVConnectorOutput) -> None: - assert self.connector_scheduler is not None - self.connector_scheduler.update_connector_output(connector_output) - - def request_finished_all_groups( - self, - request: Request, - block_ids: tuple[list[int], ...], - ) -> tuple[bool, dict[str, Any] | None]: - assert self.connector_scheduler is not None - self.connector_scheduler.requests.pop(request.request_id, None) - return False, None diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/hisparse/stats.py b/vllm/distributed/kv_transfer/kv_connector/v1/hisparse/stats.py deleted file mode 100644 index 9c7e875097..0000000000 --- a/vllm/distributed/kv_transfer/kv_connector/v1/hisparse/stats.py +++ /dev/null @@ -1,214 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Stats and Prometheus metrics for the HiSparse connector.""" - -from dataclasses import dataclass -from typing import Any - -from vllm.config import VllmConfig -from vllm.distributed.kv_transfer.kv_connector.v1.metrics import ( - KVConnectorPromMetrics, - KVConnectorStats, - PromMetric, - PromMetricT, -) -from vllm.v1.metrics.buckets import histogram_buckets -from vllm.v1.metrics.stats import KVCacheEvictionEvent -from vllm.v1.metrics.utils import create_metric_per_engine - -_HISPARSE_LEVEL_KEYS = frozenset({"host_cache_usage_perc", "pending_page_transfers"}) - - -@dataclass -class HiSparseKVConnectorStats(KVConnectorStats): - """Container for HiSparse hot-buffer residency metrics. - - Each list entry is the delta recorded over one device counter snapshot - interval, so a list can hold multiple snapshots per logging interval. - The host-tier gauges are level values, not deltas: aggregation keeps - only the most recent observation. - """ - - def __post_init__(self): - if not self.data: - # Empty container init, no data is passed in. - self.reset() - - def reset(self): - # Must be serializable - self.data: dict[str, list[int | float]] = { - "cache_hits": [], - "cache_misses": [], - "host_to_device_bytes": [], - "host_cache_usage_perc": [], - "pending_page_transfers": [], - "host_block_lifetime_seconds": [], - "host_block_idle_before_evict_seconds": [], - "host_block_reuse_gap_seconds": [], - } - - def record_snapshot(self, hits: int, misses: int, host_to_device_bytes: int): - self.data["cache_hits"].append(hits) - self.data["cache_misses"].append(misses) - self.data["host_to_device_bytes"].append(host_to_device_bytes) - - def record_host_usage(self, usage: float, pending_page_transfers: int) -> None: - self.data["host_cache_usage_perc"].append(usage) - self.data["pending_page_transfers"].append(pending_page_transfers) - - def record_host_evictions(self, events: list[KVCacheEvictionEvent]) -> None: - for event in events: - self.data["host_block_lifetime_seconds"].append(event.lifetime_seconds) - self.data["host_block_idle_before_evict_seconds"].append(event.idle_seconds) - self.data["host_block_reuse_gap_seconds"].extend(event.reuse_gaps_seconds) - - def aggregate(self, other: KVConnectorStats) -> KVConnectorStats: - if not other.is_empty(): - for k, v in other.data.items(): - accumulator = self.data[k] - assert isinstance(accumulator, list) - if k in _HISPARSE_LEVEL_KEYS: - accumulator.extend(v[-1:]) - else: - accumulator.extend(v) - return self - - def reduce(self) -> dict[str, int | float]: - # Compute compact representative stats suitable for CLI logging. - reduced: dict[str, int | float] = { - "HiSparse hot-buffer hits": sum(self.data["cache_hits"]), - "HiSparse hot-buffer misses": sum(self.data["cache_misses"]), - "HiSparse host-to-device bytes": sum(self.data["host_to_device_bytes"]), - } - if self.data["host_cache_usage_perc"]: - usage = self.data["host_cache_usage_perc"][-1] - reduced["HiSparse host KV cache usage %"] = round(100 * usage, 1) - return reduced - - def is_empty(self) -> bool: - return all(len(values) == 0 for values in self.data.values()) - - -class HiSparsePromMetrics(KVConnectorPromMetrics): - def __init__( - self, - vllm_config: VllmConfig, - metric_types: dict[type[PromMetric], type[PromMetricT]], - labelnames: list[str], - per_engine_labelvalues: dict[int, list[object]], - ): - super().__init__(vllm_config, metric_types, labelnames, per_engine_labelvalues) - - counter_cache_hits = self._counter_cls( - name="vllm:hisparse_cache_hits", - documentation="Number of HiSparse device hot-buffer hits.", - labelnames=labelnames, - ) - counter_cache_misses = self._counter_cls( - name="vllm:hisparse_cache_misses", - documentation="Number of HiSparse device hot-buffer misses.", - labelnames=labelnames, - ) - counter_host_to_device_bytes = self._counter_cls( - name="vllm:hisparse_host_to_device_bytes", - documentation=( - "Bytes transferred from host KV storage to HiSparse hot buffers." - ), - labelnames=labelnames, - ) - self.hisparse_counters: dict[str, Any] = { - "cache_hits": create_metric_per_engine( - counter_cache_hits, self.per_engine_labelvalues - ), - "cache_misses": create_metric_per_engine( - counter_cache_misses, self.per_engine_labelvalues - ), - "host_to_device_bytes": create_metric_per_engine( - counter_host_to_device_bytes, self.per_engine_labelvalues - ), - } - - gauge_host_cache_usage = self._gauge_cls( - name="vllm:hisparse_host_cache_usage_perc", - documentation=( - "HiSparse host KV-cache usage. 1 means 100 percent usage. " - "Evictable cached blocks count as free." - ), - multiprocess_mode="mostrecent", - labelnames=labelnames, - ) - gauge_pending_page_transfers = self._gauge_cls( - name="vllm:hisparse_pending_page_transfers", - documentation=( - "HiSparse page transfers (write-backs and restores) not yet completed." - ), - multiprocess_mode="mostrecent", - labelnames=labelnames, - ) - self.hisparse_gauges: dict[str, Any] = { - "host_cache_usage_perc": create_metric_per_engine( - gauge_host_cache_usage, self.per_engine_labelvalues - ), - "pending_page_transfers": create_metric_per_engine( - gauge_pending_page_transfers, self.per_engine_labelvalues - ), - } - - self.hisparse_histograms: dict[str, Any] = {} - if vllm_config.observability_config.kv_cache_metrics: - residency_buckets = histogram_buckets( - "kv_cache_residency", - overrides=vllm_config.observability_config.custom_histogram_buckets, - ) - histogram_host_block_lifetime = self._histogram_cls( - name="vllm:hisparse_host_block_lifetime_seconds", - documentation=( - "Histogram of HiSparse host KV block lifetime from allocation " - "to eviction. Sampled metrics (controlled by " - "--kv-cache-metrics-sample)." - ), - buckets=residency_buckets, - labelnames=labelnames, - ) - histogram_host_block_idle_before_evict = self._histogram_cls( - name="vllm:hisparse_host_block_idle_before_evict_seconds", - documentation=( - "Histogram of HiSparse host KV block idle time before eviction. " - "Sampled metrics (controlled by --kv-cache-metrics-sample)." - ), - buckets=residency_buckets, - labelnames=labelnames, - ) - histogram_host_block_reuse_gap = self._histogram_cls( - name="vllm:hisparse_host_block_reuse_gap_seconds", - documentation=( - "Histogram of time gaps between consecutive HiSparse host KV " - "block accesses. Sampled metrics (controlled by " - "--kv-cache-metrics-sample)." - ), - buckets=residency_buckets, - labelnames=labelnames, - ) - self.hisparse_histograms = { - "host_block_lifetime_seconds": create_metric_per_engine( - histogram_host_block_lifetime, self.per_engine_labelvalues - ), - "host_block_idle_before_evict_seconds": create_metric_per_engine( - histogram_host_block_idle_before_evict, self.per_engine_labelvalues - ), - "host_block_reuse_gap_seconds": create_metric_per_engine( - histogram_host_block_reuse_gap, self.per_engine_labelvalues - ), - } - - def observe(self, transfer_stats_data: dict[str, Any], engine_idx: int = 0): - for name, counter in self.hisparse_counters.items(): - for value in transfer_stats_data.get(name, []): - counter[engine_idx].inc(value) - for name, gauge in self.hisparse_gauges.items(): - values = transfer_stats_data.get(name, []) - if values: - gauge[engine_idx].set(values[-1]) - for name, histogram in self.hisparse_histograms.items(): - for value in transfer_stats_data.get(name, []): - histogram[engine_idx].observe(value) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/hisparse/worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/hisparse/worker.py deleted file mode 100644 index 6e3d3415c3..0000000000 --- a/vllm/distributed/kv_transfer/kv_connector/v1/hisparse/worker.py +++ /dev/null @@ -1,978 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""HiSparse worker-side host/hot state and data movement.""" - -from __future__ import annotations - -from collections import deque -from collections.abc import Mapping, Sequence -from dataclasses import dataclass -from functools import partial -from typing import TYPE_CHECKING, Any, NamedTuple - -import numpy as np -import torch - -from vllm import _custom_ops as ops -from vllm.config import VllmConfig -from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.stats import ( - HiSparseKVConnectorStats, -) -from vllm.distributed.parallel_state import ( - get_tensor_model_parallel_rank, - get_tp_group, -) -from vllm.utils.gpu_sync_debug import gpu_sync_allowed -from vllm.utils.torch_utils import current_stream -from vllm.v1.core.kv_cache_utils import KVCacheBlockCopy -from vllm.v1.hisparse.layout import HISPARSE_HOT_SUFFIX -from vllm.v1.hisparse.runtime import ( - HiSparseCacheHandle, - drop_hisparse_host_pool, - release_pinned_state, - update_hisparse_residency, -) -from vllm.v1.hisparse.types import SparseKVPageTransfer, SparseKVRowMirror -from vllm.v1.kv_cache_interface import ( - HiSparseHotSpec, - KVCacheConfig, -) -from vllm.v1.worker.utils import copy_kv_cache_blocks_inplace - -if TYPE_CHECKING: - from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.connector import ( - HiSparseConnectorMetadata, - ) - from vllm.v1.kv_offload.cpu.shared_offload_region import SharedOffloadRegion - - -class _DMADescriptors(NamedTuple): - src: torch.Tensor - dst: torch.Tensor - sizes: torch.Tensor - src_np: np.ndarray - dst_np: np.ndarray - sizes_np: np.ndarray - - -@dataclass -class _SlotMappingStaging: - stream: torch.Stream - event: torch.Event - slots: torch.Tensor - candidates: tuple[SparseKVRowMirror, ...] = () - num_tokens: int = 0 - source_index: int = 0 - - -def _allocate_dma_descriptors(size: int) -> _DMADescriptors: - src, dst, sizes = (torch.empty(size, dtype=torch.int64) for _ in range(3)) - return _DMADescriptors( - src, - dst, - sizes, - src.numpy(), - dst.numpy(), - sizes.numpy(), - ) - - -_METRICS_INTERVAL = 2000 - - -def _get_hisparse_cache( - forward_context: dict[str, Any], layer_name: str -) -> HiSparseCacheHandle: - attention_layer = forward_context[layer_name] - hisparse_cache = attention_layer.hisparse_cache - assert hisparse_cache is not None - return hisparse_cache - - -def _flatten_row_mirrors( - row_mirrors: Mapping[str, tuple[SparseKVRowMirror, ...]], - request_ids: Sequence[str] | None, -) -> tuple[SparseKVRowMirror, ...]: - ordered_ids = row_mirrors if request_ids is None else request_ids - return tuple( - mirror - for request_id in ordered_ids - for mirror in row_mirrors.get(request_id, ()) - ) - - -def _select_written_row_mirrors( - candidates: tuple[SparseKVRowMirror, ...], - source_slots: np.ndarray, - source_index: int, -) -> tuple[SparseKVRowMirror, ...]: - """Select and coalesce GPU-written rows from a scheduler-owned envelope.""" - source_slots = source_slots[source_slots >= 0] - if source_slots.size == 0 or not candidates: - return () - starts = np.asarray([mirror.source_starts for mirror in candidates], dtype=np.int64) - counts = np.fromiter((mirror.num_rows for mirror in candidates), dtype=np.int64) - destinations = np.fromiter( - [mirror.destination_start for mirror in candidates], dtype=np.int64 - ) - if source_index >= starts.shape[1]: - raise RuntimeError("HiSparse row-mirror source index is out of range.") - source_starts = starts[:, source_index] - order = np.argsort(source_starts) - matches = np.searchsorted(source_starts[order], source_slots, side="right") - 1 - candidate_ids = order[np.maximum(matches, 0)] - offsets = source_slots - source_starts[candidate_ids] - valid = (matches >= 0) & (offsets < counts[candidate_ids]) - if not np.all(valid): - # A page the scheduler could not map has no host destination, so its - # rows are simply not mirrorable this step. - source_slots = source_slots[valid] - candidate_ids = candidate_ids[valid] - offsets = offsets[valid] - if source_slots.size == 0: - return () - sources = starts[candidate_ids] + offsets[:, None] - destinations = destinations[candidate_ids] + offsets - boundaries = ( - np.flatnonzero( - (candidate_ids[1:] != candidate_ids[:-1]) - | (np.diff(destinations) != 1) - | np.any(np.diff(sources, axis=0) != 1, axis=1) - ) - + 1 - ) - run_starts = np.concatenate(([0], boundaries)) - run_ends = np.concatenate((boundaries, [destinations.size])) - return tuple( - SparseKVRowMirror( - source_starts=tuple(int(value) for value in sources[start]), - destination_start=int(destinations[start]), - num_rows=int(end - start), - ) - for start, end in zip(run_starts, run_ends, strict=True) - ) - - -def _is_hisparse_host_writer( - shared_host_region: SharedOffloadRegion | None, -) -> bool: - return shared_host_region is None or get_tensor_model_parallel_rank() == 0 - - -def _create_hisparse_host_events( - shared_host_region: SharedOffloadRegion | None, - is_host_writer: bool, - device: torch.device, -) -> tuple[torch.Event, torch.Event]: - if shared_host_region is None: - return torch.Event(), torch.Event() - - events: tuple[torch.Event, torch.Event] | None = None - ipc_handles = None - if is_host_writer: - events = ( - torch.cuda.Event(interprocess=True), - torch.cuda.Event(interprocess=True), - ) - stream = current_stream() - for event in events: - event.record(stream) - ipc_handles = tuple(event.ipc_handle() for event in events) - ipc_handles = get_tp_group().broadcast_object(ipc_handles, src=0) - if events is None: - events = ( - torch.cuda.Event.from_ipc_handle(device, ipc_handles[0]), - torch.cuda.Event.from_ipc_handle(device, ipc_handles[1]), - ) - return events - - -class HiSparseConnectorWorker: - """Own HiSparse host/hot state and execute its worker-side transfers.""" - - def __init__(self, vllm_config: VllmConfig, kv_cache_config: KVCacheConfig) -> None: - self.vllm_config = vllm_config - self.kv_cache_config = kv_cache_config - self._initialized = False - - def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]) -> None: - forward_context = self.vllm_config.compilation_config.static_forward_context - cache_handles: list[HiSparseCacheHandle] = [] - cache_layer_names: list[str] = [] - for group in self.kv_cache_config.kv_cache_groups: - if not isinstance(group.kv_cache_spec, HiSparseHotSpec): - continue - for cache_name in group.layer_names: - assert cache_name.endswith(HISPARSE_HOT_SUFFIX) - layer_name = cache_name[: -len(HISPARSE_HOT_SUFFIX)] - cache_handles.append(_get_hisparse_cache(forward_context, layer_name)) - cache_layer_names.append(layer_name) - - if not cache_handles: - raise RuntimeError("HiSparse connector found no hot-cache handles.") - hot_backings: dict[int, torch.Tensor] = {} - registered_host_pools: dict[int, torch.Tensor] = {} - shared_host_regions: dict[int, SharedOffloadRegion] = {} - for cache in cache_handles: - hot_backing = cache.runtime.hot_backing - hot_backings[hot_backing.untyped_storage().data_ptr()] = hot_backing - registered_pool = cache.runtime.registered_host_pool - registered_host_pools[registered_pool.data_ptr()] = registered_pool - if (region := cache.runtime.shared_host_region) is not None: - shared_host_regions[id(region)] = region - if len(hot_backings) != 1: - raise RuntimeError("HiSparse hot tensors must share one GPU backing.") - hot_backing = next(iter(hot_backings.values())) - if len(shared_host_regions) > 1: - raise RuntimeError("HiSparse caches must share one host region.") - shared_host_region = next(iter(shared_host_regions.values()), None) - pinned_host_pools = ( - [] - if shared_host_region is not None - else list(registered_host_pools.values()) - ) - is_host_writer = _is_hisparse_host_writer(shared_host_region) - - resident = cache_handles[0].view - assert resident is not None - host_num_blocks = self.kv_cache_config.hisparse_host_num_blocks - assert host_num_blocks is not None - try: - self.initialize( - cache_handles, - cache_layer_names, - hot_backing, - self.vllm_config.scheduler_config.max_num_seqs, - host_num_blocks, - hot_backing.device, - pinned_host_pools, - shared_host_region=shared_host_region, - is_host_writer=is_host_writer, - ) - except Exception: - release_pinned_state( - [cache.runtime for cache in cache_handles], - pinned_host_pools, - shared_host_region, - ) - drop_hisparse_host_pool(self.vllm_config) - raise - - def initialize( - self, - cache_handles: list[HiSparseCacheHandle], - cache_layer_names: list[str], - hot_backing: torch.Tensor, - max_num_reqs: int, - host_num_blocks: int, - device: torch.device, - pinned_host_pools: list[torch.Tensor], - *, - shared_host_region: SharedOffloadRegion | None = None, - is_host_writer: bool = True, - ) -> None: - if self._initialized: - raise RuntimeError("HiSparse connector worker is already initialized.") - resident = cache_handles[0].view - assert resident is not None - self.kernel_block_size = resident.block_size - self.host_num_blocks = host_num_blocks - self.pinned_host_pools = pinned_host_pools - self.shared_host_region = shared_host_region - self.is_host_writer = is_host_writer - self.dma_stream = ( - torch.cuda.Stream(device=device) if self.is_host_writer else None - ) - self._slot_mapping_staging = None - if self.is_host_writer: - max_mirror_rows = ( - self.vllm_config.scheduler_config.max_num_batched_tokens - + max_num_reqs * (self.vllm_config.num_lookahead_tokens + 1) - ) - self._slot_mapping_staging = _SlotMappingStaging( - stream=torch.Stream(device=device), - event=torch.Event(), - slots=torch.empty( - max_mirror_rows, - dtype=torch.int64, - pin_memory=True, - ), - ) - self._dma_free_descriptors: list[_DMADescriptors] = [] - self._pending_dma_descriptors: deque[tuple[torch.Event, _DMADescriptors]] = ( - deque() - ) - self._dma_submitted = False - self._per_layer_mirrored: set[int] = set() - self._submitted_mirror_layers: set[int] = set() - self._layer_ready_events = tuple(torch.Event() for _ in cache_handles) - self._forward_ready_event = torch.Event() - self._draft_layers = tuple( - index for index, cache in enumerate(cache_handles) if cache.draft_layer - ) - self._draft_mirror_pending = False - self._set_row_mirrors(()) - self.cache_layer_names = cache_layer_names - self._group_leaders = tuple( - (layer_name, cache) - for layer_name, cache in zip(cache_layer_names, cache_handles, strict=True) - if cache.runtime.is_group_leader - ) - self.host_write_events = _create_hisparse_host_events( - shared_host_region, is_host_writer, device - ) - self.host_write_event = self.host_write_events[1] - self._next_host_write_event = 0 - self.cache_handles = cache_handles - self.leader_runtimes = [cache.runtime for _, cache in self._group_leaders] - request_state_indices = { - indices.data_ptr(): indices - for cache in cache_handles - if (indices := cache.runtime.request_state_indices) is not None - } - if len(request_state_indices) != 1: - raise RuntimeError( - "HiSparse runtimes must share one request-state mapping." - ) - self.request_state_indices = next(iter(request_state_indices.values())) - if self.request_state_indices.numel() != max_num_reqs: - raise RuntimeError( - "HiSparse request-state mapping does not match max_num_seqs." - ) - self.hot_backing = hot_backing - self.residency: torch.Tensor | None = next( - (cache.residency for cache in cache_handles if cache.residency is not None), - None, - ) - self._pending_invalid_block_ids: list[int] = [] - # Destination block ids of host copies this worker has run. - self._completed_host_copy_dst_ids: list[int] = [] - self._post_forward_transfers: list[SparseKVPageTransfer] = [] - self._enqueued_transfer_ids: list[int] = [] - self._pending_transfer_events: deque[tuple[torch.Event, tuple[int, ...]]] = ( - deque() - ) - self._metrics_calls = 0 - self._metrics_event = torch.Event() - self._metrics_pending = False - self._init_dma() - if self.is_host_writer: - for layer_index, handle in enumerate(cache_handles): - # Draft layers are mirrored once in the next start_step instead. - if not handle.draft_layer: - handle.submit_layer_mirror = partial( - self._enqueue_layer_mirror, layer_index - ) - self._initialized = True - - def set_request_state_indices(self, indices: torch.Tensor) -> None: - if indices.numel() > self.request_state_indices.numel(): - raise ValueError( - "HiSparse request-state mapping exceeds max_num_seqs: " - f"{indices.numel()} > {self.request_state_indices.numel()}." - ) - if torch.cuda.is_current_stream_capturing(): - return - # Attention indexes persistent request state by input-batch row. Refresh - # that indirection after every batch compaction or reorder. - self.request_state_indices.fill_(-1) - self.request_state_indices[: indices.numel()].copy_(indices) - if self._pending_invalid_block_ids: - self.invalidate_blocks(self._pending_invalid_block_ids, indices) - self._pending_invalid_block_ids.clear() - - def _init_dma(self) -> None: - host_caches = tuple(cache.runtime.host_cache for cache in self.cache_handles) - resident_caches = [] - for cache in self.cache_handles: - assert cache.view is not None and cache.slot_mapping is not None - resident_caches.append(cache.view.cache) - self.host_caches = host_caches - self.resident_caches = tuple(resident_caches) - - for resident_cache, host_cache in zip(resident_caches, host_caches): - row_bytes = resident_cache.shape[-1] * resident_cache.element_size() - if ( - resident_cache.ndim != 3 - or resident_cache.shape[1] != self.kernel_block_size - or resident_cache.stride(1) * resident_cache.element_size() != row_bytes - ): - raise RuntimeError("HiSparse DMA requires contiguous resident rows.") - if ( - host_cache.ndim != 2 - or not host_cache.is_contiguous() - or host_cache.shape[1] * host_cache.element_size() != row_bytes - ): - raise RuntimeError("HiSparse DMA requires contiguous host rows.") - - def start_step( - self, - metadata: HiSparseConnectorMetadata, - request_state_indices: torch.Tensor | None, - request_ids: list[str] | None = None, - num_tokens: int = 0, - ) -> None: - self._stage_row_mirror_mapping(num_tokens) - self._finish_previous_step() - previous_host_write_event = self.host_write_event - self.host_write_event = self.host_write_events[self._next_host_write_event] - self._next_host_write_event ^= 1 - current_stream().wait_event(previous_host_write_event) - if self.shared_host_region is not None and not num_tokens: - # Without a forward, no TP collective keeps rank 0 from recording - # a shared event again before every rank has issued its wait. - get_tp_group().barrier() - self._release_completed_dma_descriptors() - self._dma_submitted = False - mirrors = _flatten_row_mirrors(metadata.row_mirrors, request_ids) - if self._slot_mapping_staging is not None: - self._slot_mapping_staging.candidates = mirrors - self._set_row_mirrors(mirrors) - self._clear_forward_mirror_state() - for handle in self.cache_handles: - handle.all_context_pages_resident = metadata.all_context_pages_resident - self._copy_host_blocks(metadata.host_block_copies, previous_host_write_event) - transfers = ( - metadata.command.page_transfers if metadata.command is not None else [] - ) - self._restore_pages([transfer for transfer in transfers if transfer.restore]) - self._post_forward_transfers = [ - transfer - for transfer in transfers - if not transfer.restore and transfer.after_forward - ] - self._submit_transfers( - [ - transfer - for transfer in transfers - if not transfer.restore and not transfer.after_forward - ] - ) - self._pending_invalid_block_ids.extend(metadata.source_block_ids) - if request_state_indices is not None: - self.set_request_state_indices(request_state_indices) - if metadata.residency_updates: - assert self.residency is not None - assert request_state_indices is not None and request_ids is not None - update_hisparse_residency( - self.residency, - metadata.residency_updates, - request_ids, - request_state_indices, - ) - - def _clear_forward_mirror_state(self) -> None: - self._per_layer_mirrored.clear() - self._submitted_mirror_layers.clear() - for handle in self.cache_handles: - handle.decode_batch = False - handle.host_mirror_required = False - handle.num_actual_tokens = 0 - handle.num_decode_tokens = 0 - handle.req_id_per_token = None - - def prepare_forward(self, attn_metadata: Mapping[str, Any] | None) -> None: - if attn_metadata is None: - return - for layer_name, handle in self._group_leaders: - metadata = attn_metadata.get(layer_name) - if metadata is not None: - handle.prepare_group_for_batch(metadata) - - def _stage_row_mirror_mapping(self, num_tokens: int) -> None: - """Snapshot the rows this forward will write, off the compute stream. - - The resident slot mapping is a persistent view bound at registration - time, so the rows are already staged by the time the forward launches. - """ - state = self._slot_mapping_staging - if state is None or not num_tokens: - return - handle = self.cache_handles[0] - slots = handle.slot_mapping - assert slots is not None - source_index = handle.runtime.resident_source_index - start = state.num_tokens - end = start + num_tokens - if end > state.slots.shape[0]: - raise ValueError( - "HiSparse row mapping exceeds staging capacity: " - f"{end} > {state.slots.shape[0]}." - ) - main_stream = current_stream() - state.stream.wait_stream(main_stream) - with torch.cuda.stream(state.stream): - state.slots[start:end].copy_( - slots[:num_tokens], - non_blocking=True, - ) - state.event.record(state.stream) - state.num_tokens = end - state.source_index = source_index - - def _resolve_row_mirrors(self, state: _SlotMappingStaging) -> None: - """Compute the final row mirrors from the staged GPU slot mapping.""" - self._set_row_mirrors( - _select_written_row_mirrors( - state.candidates, - state.slots[: state.num_tokens].numpy(), - state.source_index, - ) - ) - state.num_tokens = 0 - - def _copy_host_blocks( - self, - host_block_copies: Sequence[KVCacheBlockCopy], - previous_host_write_event: torch.Event, - ) -> None: - if not host_block_copies: - return - self._completed_host_copy_dst_ids.extend( - copy.dst_block_id for copy in host_block_copies - ) - if self.shared_host_region is None or get_tensor_model_parallel_rank() == 0: - if self.host_caches: - previous_host_write_event.synchronize() - copy_kv_cache_blocks_inplace( - self.host_caches, - self.host_num_blocks, - host_block_copies, - ) - if self.shared_host_region is not None: - get_tp_group().barrier() - - def invalidate_blocks( - self, block_ids: list[int], request_state_indices: torch.Tensor - ) -> None: - """Invalidate recycled host slots in this worker's leader runtimes.""" - if not block_ids: - return - device = self.cache_handles[0].runtime.device - staging = torch.tensor(block_ids, dtype=torch.int32, pin_memory=True) - blocks = staging.to(device, dtype=torch.int32, non_blocking=True) - offsets = torch.arange(self.kernel_block_size, dtype=torch.int32, device=device) - slots = (blocks[:, None] * self.kernel_block_size + offsets[None, :]).flatten() - sorted_slots = torch.sort(slots).values - state_indices = request_state_indices.to(device=device, dtype=torch.long) - for runtime in self.leader_runtimes: - runtime.invalidate_sorted_slots(sorted_slots, state_indices) - - def reset_hot_state(self) -> None: - for runtime in self.leader_runtimes: - runtime.reset_hot_state() - - def get_kv_connector_stats(self) -> HiSparseKVConnectorStats | None: - stats = None - if self._metrics_pending and self._metrics_event.query(): - stats = HiSparseKVConnectorStats() - for runtime in self.leader_runtimes: - group = runtime.index_group - hits, misses = group.swap_stats_host.tolist() - if hits or misses: - stats.record_snapshot(hits, misses, misses * group.stats_row_bytes) - self._metrics_pending = False - if stats.is_empty(): - stats = None - - self._metrics_calls += 1 - if ( - self._metrics_calls % _METRICS_INTERVAL == 0 - and not self._metrics_pending - and not torch.cuda.is_current_stream_capturing() - ): - compute_stream = current_stream() - for runtime in self.leader_runtimes: - group = runtime.index_group - compute_stream.wait_stream(group.copy_stream) - group.swap_stats_host.copy_(group.swap_stats, non_blocking=True) - group.swap_stats.zero_() - group.copy_stream.wait_stream(compute_stream) - self._metrics_event.record() - self._metrics_pending = True - return stats - - def _release_completed_dma_descriptors(self) -> None: - pending = self._pending_dma_descriptors - while pending and pending[0][0].query(): - _, descriptors = pending.popleft() - self._dma_free_descriptors.append(descriptors) - - def _acquire_dma_descriptors(self, size: int) -> _DMADescriptors: - for index, descriptors in enumerate(self._dma_free_descriptors): - if descriptors.src.numel() >= size: - return self._dma_free_descriptors.pop(index) - return _allocate_dma_descriptors(size) - - def _submit_dma_descriptors( - self, - descriptors: _DMADescriptors, - descriptor_count: int, - transfer_ids: tuple[int, ...] = (), - ready_event: torch.Event | None = None, - ) -> None: - stream = self.dma_stream - assert stream is not None - if ready_event is None: - stream.wait_stream(current_stream()) - else: - stream.wait_event(ready_event) - completion_event = torch.Event() - with torch.cuda.stream(stream): - ops.swap_blocks_batch( - descriptors.src[:descriptor_count], - descriptors.dst[:descriptor_count], - descriptors.sizes[:descriptor_count], - ) - self.host_write_event.record(stream) - completion_event.record(stream) - self._pending_dma_descriptors.append((completion_event, descriptors)) - self._dma_submitted = True - if transfer_ids: - self._pending_transfer_events.append((completion_event, transfer_ids)) - self._enqueued_transfer_ids.extend(transfer_ids) - - def _set_row_mirrors(self, mirrors: tuple[SparseKVRowMirror, ...]) -> None: - self._row_mirrors = mirrors - self._row_mirror_destination_starts = np.fromiter( - (mirror.destination_start for mirror in mirrors), - dtype=np.int64, - count=len(mirrors), - ) - self._row_mirror_counts = np.fromiter( - (mirror.num_rows for mirror in mirrors), - dtype=np.int64, - count=len(mirrors), - ) - if mirrors: - self._row_mirror_source_starts = np.asarray( - [mirror.source_starts for mirror in mirrors], dtype=np.int64 - ) - if self._row_mirror_source_starts.ndim != 2: - raise RuntimeError("HiSparse DMA source mappings must be rectangular.") - else: - self._row_mirror_source_starts = np.empty((0, 0), dtype=np.int64) - self._row_mirror_num_rows = int(self._row_mirror_counts.sum()) - - def _enqueue_row_dma( - self, layer_indices: Sequence[int], ready_event: torch.Event | None = None - ) -> None: - if ( - not layer_indices - or not self._row_mirror_num_rows - or not self.is_host_writer - ): - return - mirrors = self._row_mirrors - num_layers = len(layer_indices) - descriptor_count = len(mirrors) * num_layers - descriptors = self._acquire_dma_descriptors(descriptor_count) - destination_starts = self._row_mirror_destination_starts - row_counts = self._row_mirror_counts - for descriptor_offset, layer_index in enumerate(layer_indices): - cache = self.cache_handles[layer_index] - source_index = cache.runtime.resident_source_index - if source_index >= self._row_mirror_source_starts.shape[1]: - raise RuntimeError("HiSparse row DMA source index is out of range.") - source_rows = self._row_mirror_source_starts[:, source_index] - source = self.resident_caches[layer_index] - destination = self.host_caches[layer_index] - row_bytes = source.shape[-1] * source.element_size() - if ( - source.stride(1) * source.element_size() != row_bytes - or destination.shape[1] * destination.element_size() != row_bytes - ): - raise RuntimeError("HiSparse row DMA requires contiguous rows.") - source_blocks, source_row_offsets = np.divmod( - source_rows, self.kernel_block_size - ) - source_out_of_range = ( - np.any(source_blocks < 0) - or np.any(source_blocks >= source.shape[0]) - or np.any(source_row_offsets + row_counts > self.kernel_block_size) - ) - if source_out_of_range: - raise RuntimeError("HiSparse row DMA source is out of range.") - if np.any(destination_starts < 0) or np.any( - destination_starts + row_counts > destination.shape[0] - ): - raise RuntimeError("HiSparse row DMA index is out of range.") - descriptor_slice = slice(descriptor_offset, descriptor_count, num_layers) - descriptors.src_np[descriptor_slice] = ( - source.data_ptr() - + source_blocks * source.stride(0) * source.element_size() - + source_row_offsets * row_bytes - ) - descriptors.dst_np[descriptor_slice] = ( - destination.data_ptr() + destination_starts * row_bytes - ) - descriptors.sizes_np[descriptor_slice] = row_counts * row_bytes - - self._submit_dma_descriptors( - descriptors, descriptor_count, ready_event=ready_event - ) - - def _enqueue_layer_mirror(self, layer_index: int) -> None: - handle = self.cache_handles[layer_index] - if not handle.host_mirror_required: - return - if layer_index in self._per_layer_mirrored: - raise RuntimeError(f"HiSparse layer {layer_index} mirrored twice.") - state = self._slot_mapping_staging - if state is not None and state.num_tokens: - # The staging stream waited on the compute stream before the - # forward launched, so this only drains pre-forward work that is - # already queued ahead of the layer kernels: no GPU bubble. - state.event.synchronize() - self._resolve_row_mirrors(state) - self._per_layer_mirrored.add(layer_index) - next_layer = layer_index + 1 - if ( - next_layer < len(self.cache_handles) - and self.cache_handles[next_layer].runtime.resident_source_index - == handle.runtime.resident_source_index - ): - return - ready_event = self._layer_ready_events[layer_index] - ready_event.record() - pending_layers = tuple( - sorted(self._per_layer_mirrored - self._submitted_mirror_layers) - ) - self._enqueue_row_dma( - pending_layers, - ready_event=ready_event, - ) - self._submitted_mirror_layers.update(pending_layers) - - def _record_transfer_completion( - self, - transfers: list[SparseKVPageTransfer], - *, - stream: torch.Stream | None = None, - ) -> None: - if not transfers: - return - if stream is None: - if not self.is_host_writer: - return - stream = self.dma_stream - assert stream is not None - completion_event = torch.Event() - completion_event.record(stream) - transfer_ids = tuple(transfer.transfer_id for transfer in transfers) - self._pending_transfer_events.append((completion_event, transfer_ids)) - self._enqueued_transfer_ids.extend(transfer_ids) - - def _restore_pages(self, transfers: list[SparseKVPageTransfer]) -> None: - """Restore imported tails on every rank before its forward or graph replay.""" - if not transfers: - return - for layer_index, cache in enumerate(self.cache_handles): - source_index = cache.runtime.resident_source_index - source = self.host_caches[layer_index] - destination = self.resident_caches[layer_index] - for transfer in transfers: - host_start = transfer.host_block_id * self.kernel_block_size - resident_block = transfer.resident_block_ids[source_index] - # Each import pays one H2D page copy and enqueue per layer/rank - # instead of tail prefill on D. Restore the prompt rows before - # decode appends to this page; unused rows do not extend the - # logical sequence. - # cudaHostRegister pins this pool outside PyTorch's allocator, - # so the sync checker cannot recognize it via is_pinned(). - with gpu_sync_allowed(): - destination[resident_block].copy_( - source[host_start : host_start + self.kernel_block_size], - non_blocking=True, - ) - self._record_transfer_completion(transfers, stream=current_stream()) - - def _submit_transfers(self, transfers: list[SparseKVPageTransfer]) -> None: - if self.cache_handles[0].runtime.eager_host_mirror: - self._record_transfer_completion(transfers) - else: - self._enqueue_transfers(transfers) - - def _enqueue_transfers(self, transfers: list[SparseKVPageTransfer]) -> None: - if not transfers or not self.is_host_writer: - return - num_layers = len(self.cache_handles) - descriptor_count = len(transfers) * num_layers - descriptors = self._acquire_dma_descriptors(descriptor_count) - destination_rows = np.fromiter( - (transfer.host_block_id * self.kernel_block_size for transfer in transfers), - dtype=np.int64, - count=len(transfers), - ) - source_blocks_by_transfer = np.asarray( - [transfer.resident_block_ids for transfer in transfers], dtype=np.int64 - ) - if source_blocks_by_transfer.ndim != 2: - raise RuntimeError( - "HiSparse spill DMA source mappings must be rectangular." - ) - for layer_index, cache in enumerate(self.cache_handles): - source_index = cache.runtime.resident_source_index - if source_index >= source_blocks_by_transfer.shape[1]: - raise RuntimeError("HiSparse spill DMA source index is out of range.") - source_blocks = source_blocks_by_transfer[:, source_index] - source = self.resident_caches[layer_index] - destination = self.host_caches[layer_index] - if np.any(source_blocks < 0) or np.any(source_blocks >= source.shape[0]): - raise RuntimeError("HiSparse spill DMA source is out of range.") - if np.any(destination_rows < 0) or np.any( - destination_rows + self.kernel_block_size > destination.shape[0] - ): - raise RuntimeError("HiSparse spill DMA destination is out of range.") - row_bytes = source.shape[-1] * source.element_size() - descriptor_slice = slice(layer_index, descriptor_count, num_layers) - descriptors.src_np[descriptor_slice] = ( - source.data_ptr() - + source_blocks * source.stride(0) * source.element_size() - ) - descriptors.dst_np[descriptor_slice] = ( - destination.data_ptr() + destination_rows * row_bytes - ) - descriptors.sizes_np[descriptor_slice] = self.kernel_block_size * row_bytes - self._submit_dma_descriptors( - descriptors, - descriptor_count, - transfer_ids=tuple(transfer.transfer_id for transfer in transfers), - ) - - def _enqueue_host_mirror( - self, - ready_event: torch.Event | None = None, - ) -> None: - active = [ - (index, handle) - for index, handle in enumerate(self.cache_handles) - if handle.num_actual_tokens != 0 - ] - if not active: - return - - def mirror_key(handle: HiSparseCacheHandle) -> tuple: - slots = handle.mirror_slot_mapping - return ( - handle.num_actual_tokens, - handle.num_decode_tokens, - handle.decode_batch, - handle.host_mirror_required, - handle.runtime.eager_host_mirror, - None if slots is None else slots.data_ptr(), - ) - - keys = {mirror_key(handle) for _, handle in active} - if len(keys) > 1: - raise RuntimeError( - f"HiSparse cache layers disagree on mirror metadata: {list(keys)}." - ) - cache = active[0][1] - if not cache.host_mirror_required: - return - dst_slots = cache.mirror_slot_mapping - if dst_slots is None: - raise RuntimeError("HiSparse host mirror has no source slot mapping.") - num_rows = min(cache.num_actual_tokens, dst_slots.numel()) - if num_rows == 0: - return - if self.is_host_writer: - # The drafter has not written its layers' rows yet; the next - # start_step mirrors them after it runs. - self._draft_mirror_pending = bool(self._draft_layers) - expected_layers = {index for index, _ in active}.difference( - self._draft_layers - ) - if self._per_layer_mirrored and self._per_layer_mirrored != expected_layers: - raise RuntimeError( - "HiSparse per-layer DMA did not mirror every active layer: " - f"expected {sorted(expected_layers)}, got " - f"{sorted(self._per_layer_mirrored)}." - ) - pending_layers = tuple( - sorted(expected_layers - self._submitted_mirror_layers) - ) - if pending_layers: - self._enqueue_row_dma(pending_layers, ready_event=ready_event) - self._submitted_mirror_layers.update(pending_layers) - assert cache.req_id_per_token is not None - for _, handle in active: - if handle.runtime.is_group_leader: - handle.runtime.invalidate_written_slots( - dst_slots[:num_rows], - cache.req_id_per_token[:num_rows], - ) - - def _finish_mirror_phase(self, ready_event: torch.Event | None = None) -> None: - state = self._slot_mapping_staging - if state is not None and state.num_tokens: - # Measured free: blocking here costs no throughput (26.7 vs 27.6 - # gen tok/s) and lets every step mirror exactly the written rows. - state.event.synchronize() - self._resolve_row_mirrors(state) - self._enqueue_host_mirror(ready_event) - self._clear_forward_mirror_state() - - def finish_forward(self) -> None: - compute_stream = current_stream() - self._forward_ready_event.record() - self._finish_mirror_phase(self._forward_ready_event) - if self.is_host_writer: - if self._dma_submitted: - compute_stream.wait_event(self.host_write_event) - self._dma_submitted = False - else: - self.host_write_event.record(compute_stream) - self._release_completed_dma_descriptors() - - def _finish_previous_step(self) -> None: - """Mirror the previous step's draft-layer rows, then hand its pages over. - - Runs before this step's work, so after the previous step's drafter. - Post-forward transfers are ordered behind the draft mirror, so a page is - never reported clean, and its resident block never released, before - its draft-layer rows reach the host. Both land in the previous step's - host write event, which this step waits on before its forward and its - host copies, since the scheduler may already have reused the - finished requests' blocks they read and write. - """ - if self._draft_mirror_pending: - self._draft_mirror_pending = False - self._enqueue_row_dma(self._draft_layers) - transfers = self._post_forward_transfers - self._post_forward_transfers = [] - self._submit_transfers(transfers) - - def take_completed_host_copies(self) -> list[int]: - """Drain host copies this worker has enqueued for this step.""" - completed = self._completed_host_copy_dst_ids - self._completed_host_copy_dst_ids = [] - return completed - - def take_transfer_updates(self) -> tuple[list[int], list[int]]: - enqueued = self._enqueued_transfer_ids - self._enqueued_transfer_ids = [] - completed: list[int] = [] - while self._pending_transfer_events: - event, transfer_ids = self._pending_transfer_events[0] - if not event.query(): - break - self._pending_transfer_events.popleft() - completed.extend(transfer_ids) - return enqueued, completed - - def shutdown(self) -> None: - if not self._initialized: - return - if self._slot_mapping_staging is not None: - self._slot_mapping_staging.stream.synchronize() - if self.dma_stream is not None: - self.dma_stream.synchronize() - release_pinned_state( - [cache.runtime for cache in self.cache_handles], - self.pinned_host_pools, - self.shared_host_region, - ) - drop_hisparse_host_pool(self.vllm_config) - self._initialized = False diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/moriio/__init__.py b/vllm/distributed/kv_transfer/kv_connector/v1/moriio/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_common.py b/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_common.py deleted file mode 100644 index 04fd38be2f..0000000000 --- a/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_common.py +++ /dev/null @@ -1,656 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -import contextlib -import math -import os -import threading -import time -from collections.abc import Iterator -from dataclasses import dataclass -from typing import TYPE_CHECKING, Any, NamedTuple - -import msgspec -import regex as re -import torch -import zmq - -from vllm.config import KVTransferConfig, VllmConfig -from vllm.distributed.kv_transfer.kv_connector.utils import BlockIds -from vllm.distributed.kv_transfer.kv_connector.v1.base import ( - KVConnectorMetadata, -) -from vllm.distributed.parallel_state import ( - get_tensor_model_parallel_rank, - get_tensor_model_parallel_world_size, -) -from vllm.logger import init_logger -from vllm.utils.network_utils import ( - get_ip, - get_open_port, - make_zmq_socket, -) - -if TYPE_CHECKING: - pass - -from dataclasses import field -from enum import Enum - -logger = init_logger(__name__) - - -Transfer = tuple[int, float] -EngineId = str -ReqId = str -TransferId = str -TransferOffsetsKey = tuple[str, tuple[int, ...], tuple[int, ...], torch.dtype] - - -def _positive_finite_timeout(name: str, value: Any) -> float: - timeout = float(value) - if not math.isfinite(timeout) or timeout <= 0: - raise ValueError(f"{name} must be finite and greater than zero") - return timeout - - -class MoRIIOTransferAck(NamedTuple): - transfer_id: TransferId - consumer_tp_size: int = 1 - - -@dataclass -class WriteTask: - request_id: ReqId - transfer_id: TransferId - dst_engine_id: str - local_block_ids: BlockIds - remote_block_ids_hint: BlockIds | None - layer_name: str - event: torch.cuda.Event - remote_notify_port: int - remote_ip: str - remote_tp_size: int | None = None - multi_pod_hosts: list[str] = field(default_factory=list) - remote_dp_size_local: int = 0 - remote_dp_size: int = 1 - enqueue_time: float = field(default_factory=time.perf_counter) - retried: int = 0 - - -@dataclass -class LayerTransferPlan: - """Plan for transferring a single layer.""" - - request_id: ReqId - transfer_id: TransferId - layer_name: str - sess_idx: int - transfer_local_offsets: list[int] - transfer_remote_offsets: list[int] - transfer_sizes: list[int] - use_batch: bool = True - - -@dataclass -class RemoteAllocInfo: - """Information about remote block allocation.""" - - block_ids: BlockIds | None - writes_done: int = 0 - writes_expected: int | None = None - decode_dp_rank: int = 0 - completion_request_id: str | None = None - completion_notify_port: int | None = None - completion_remote_ip: str | None = None - completion_notified: bool = False - transfer_statuses: list[Any] = field(default_factory=list) - transfer_offsets: dict[ - TransferOffsetsKey, - tuple[list[int], list[int], list[int]], - ] = field(default_factory=dict) - - -class ROLE(Enum): - PRODUCER = "producer" - CONSUMER = "consumer" - NOTINIT = "notinit" - - -class MoRIIOAgentMetadata( - msgspec.Struct, - omit_defaults=True, # type: ignore[call-arg] - # required for @cached_property.d - dict=True, -): - engine_id: str - agent_metadata: bytes - kv_caches_base_addr: list[int] - num_blocks: int - block_len: int - attn_backend_name: str - - -class RoleManager: - """Manages role state across the connector.""" - - _instance: "RoleManager | None" = None - _lock = threading.Lock() - - def __init__(self) -> None: - self._role: ROLE = ROLE.NOTINIT - - @classmethod - def get_instance(cls) -> "RoleManager": - if cls._instance is None: - with cls._lock: - if cls._instance is None: - cls._instance = cls() - return cls._instance - - def set_role(self, role: ROLE) -> None: - """Set the current role.""" - with self._lock: - self._role = role - - def get_role(self) -> ROLE: - """Get the current role.""" - return self._role - - -def set_role(role: ROLE): - """Set the global role.""" - RoleManager.get_instance().set_role(role) - - -def get_role() -> ROLE: - """Get the global role.""" - return RoleManager.get_instance().get_role() - - -class MoRIIOMode(Enum): - READ = "read" - WRITE = "write" - - -class TransferBatchState(Enum): - """Verdict for a group of transfer statuses that belong to one request. - - A request's KV transfer is spread over one status per layer (two for a KDA - layer), so a single status never decides the request: PENDING means at - least one is still in flight and none has failed, FAILED means at least - one failed, DONE means all succeeded. - """ - - DONE = "done" - FAILED = "failed" - PENDING = "pending" - - -class MoRIIOError(Exception): - """Base exception for MoRIIO operations.""" - - pass - - -class HandshakeError(MoRIIOError): - """Exception raised when handshake fails.""" - - pass - - -class TransferError(MoRIIOError): - """Exception raised when transfer fails.""" - - pass - - -def get_moriio_mode(kv_transfer_config: KVTransferConfig) -> MoRIIOMode: - read_mode = str( - kv_transfer_config.kv_connector_extra_config.get("read_mode", "false") - ).lower().strip() in ("true", "1") - logger.debug("MoRIIO Connector read_mode: %s", read_mode) - if read_mode: - return MoRIIOMode.READ - else: - return MoRIIOMode.WRITE - - -def get_port_offset(dp_rank: int, tp_rank: int, tp_size: int = 1) -> int: - return (dp_rank) * tp_size + tp_rank - - -def fold_local_rank(global_dp_rank: int, dp_size_local: int) -> int: - """Fold a global DP rank into its pod-local rank [0, dp_size_local). - - ``dp_size_local == 0`` is the external-DP sentinel (local size unknown): - return the rank unchanged, since a global DP rank is always < the global - DP size so no folding is needed and the modulo is skipped (never divides - by zero). - """ - return global_dp_rank % dp_size_local if dp_size_local else global_dp_rank - - -def pod_index(global_dp_rank: int, dp_size_local: int) -> int: - """Pod index (0-based) a global DP rank lives on for Wide-EP multi-pod. - - ``dp_size_local == 0`` (external-DP sentinel) collapses to a single pod. - """ - return global_dp_rank // dp_size_local if dp_size_local else 0 - - -def resolve_host_ip(extra_config: dict) -> str: - """The IP this MoRIIO process advertises for KV transfer. - - Honors an explicit ``host_ip`` in ``kv_connector_extra_config`` before - falling back to ``get_ip()``. An external router/orchestrator can set it to - the node's routable address; this is required under frameworks (e.g. Ray) - where ``get_ip()`` resolves to an unroutable public IP and ``VLLM_HOST_IP`` - cannot be propagated to the worker processes that bind the transfer engine. - """ - return extra_config.get("host_ip") or get_ip() - - -_DEPRECATED_ENV_VARS: dict[str, str] = { - "VLLM_MORIIO_CONNECTOR_READ_MODE": "read_mode", - "VLLM_MORIIO_QP_PER_TRANSFER": "qp_per_transfer", - "VLLM_MORIIO_POST_BATCH_SIZE": "post_batch_size", - "VLLM_MORIIO_NUM_WORKERS": "num_workers", -} - - -def _warn_deprecated_env_vars() -> None: - for env_var, new_key in _DEPRECATED_ENV_VARS.items(): - if env_var in os.environ: - logger.warning_once( - "The environment variable %s is deprecated and ignored. " - "Set %r inside kv_transfer_config.kv_connector_extra_config " - "instead.", - env_var, - new_key, - ) - - -@dataclass -class MoRIIOConfig: - local_ip: str - local_kv_port: int - proxy_ip: str - local_ping_port: int - proxy_ping_port: int - http_port: int - handshake_port: int - notify_port: int - tp_rank: int - dp_rank: int - dp_size: int - tp_size: int - transfer_timeout: float - defer_timeout: float - recv_abort_timeout: float - read_mode: bool = False - qp_per_transfer: int = 1 - post_batch_size: int = -1 - num_workers: int = 1 - backend: str = "rdma" - - @classmethod - def from_vllm_config(cls, vllm_config: VllmConfig) -> "MoRIIOConfig": - # Port Configuration: - # local_ping_port -> Outgoing heartbeat to proxy - # proxy_ping_port -> Remote proxy's heartbeat ingress port - # http_port -> Instance's HTTP service endpoint - # local_kv_port -> service port for mori engine - # notify_port -> For synchronizing stages between prefill and decode - # handshake_port -> For initial handshake between mori engine - - # Optional tuning knobs - # read_mode -> If true, run the connector in READ mode (consumer - # pulls KV from producer) instead of the default - # WRITE mode. - # transfer_timeout -> Timeout for waiting_for_transfer_complete before - # raising TransferError (sec). - # defer_timeout -> Timeout for deferred WRITE allocation or a missing - # READ release notification (sec). - # recv_abort_timeout -> Timeout before an in-flight recv whose RDMA - # completion never arrived is aborted (sec). - - # Knobs for RDMA transfers, ignored if on xgmi backend - # qp_per_transfer -> Number of RDMA Queue Pairs per KV transfer. - # post_batch_size -> Batch size for posting transfer work requests - # (-1 lets the MoRI backend choose). - # num_workers -> Number of background worker threads the MoRI - # engine uses for transfer processing. - - # TODO : merge notify_port and handshake_port to simplify port management - # supports non-contiguous ports - assert vllm_config.kv_transfer_config is not None, ( - "kv_transfer_config must be set for MoRIIOConnector" - ) - _warn_deprecated_env_vars() - kv_transfer_config = vllm_config.kv_transfer_config - extra_config = kv_transfer_config.kv_connector_extra_config - tp_rank = get_tensor_model_parallel_rank() - # Fold the global data_parallel_rank back to [0, dp_size_local) for - # per-node port allocation (handles the external-DP sentinel). - pc = vllm_config.parallel_config - dp_rank = fold_local_rank(pc.data_parallel_rank, pc.data_parallel_size_local) - base_notify_port = int(extra_config["notify_port"]) - tp_size = get_tensor_model_parallel_world_size() - port_offset = get_port_offset(dp_rank, tp_rank) - backend = str(extra_config.get("backend", "rdma")).lower() - if backend not in ("rdma", "xgmi"): - raise ValueError( - f"Invalid MoRIIO backend {backend!r} in kv_connector_extra_config; " - "must be one of 'rdma' or 'xgmi'." - ) - - transfer_timeout = _positive_finite_timeout( - "kv_connector_extra_config.transfer_timeout", - extra_config.get( - "transfer_timeout", MoRIIOConstants.DEFAULT_TRANSFER_TIMEOUT - ), - ) - defer_timeout = _positive_finite_timeout( - "kv_connector_extra_config.defer_timeout", - extra_config.get("defer_timeout", MoRIIOConstants.DEFAULT_DEFER_TIMEOUT), - ) - legacy_recv_abort_timeout = os.environ.get("VLLM_MORIIO_TRANSFER_TIMEOUT_S") - if legacy_recv_abort_timeout is not None: - logger.warning_once( - "The environment variable VLLM_MORIIO_TRANSFER_TIMEOUT_S is " - "deprecated. Set 'recv_abort_timeout' inside " - "kv_transfer_config.kv_connector_extra_config instead." - ) - recv_abort_timeout = _positive_finite_timeout( - "kv_connector_extra_config.recv_abort_timeout", - extra_config.get( - "recv_abort_timeout", - legacy_recv_abort_timeout or MoRIIOConstants.DEFAULT_RECV_ABORT_TIMEOUT, - ), - ) - - return cls( - local_ip=resolve_host_ip(extra_config), - local_kv_port=get_open_port(), - proxy_ip=extra_config.get("proxy_ip", ""), - local_ping_port=get_open_port(), - proxy_ping_port=int(extra_config.get("proxy_ping_port", 0)), - http_port=int(extra_config["http_port"]), - handshake_port=int(extra_config["handshake_port"]), - notify_port=base_notify_port + port_offset, - tp_rank=tp_rank, - dp_rank=dp_rank, - # Advertise the GLOBAL DP world size to the routing proxy so it can - # address every rank across pods; dp_size_local above is only for - # per-pod port folding, not for routing. - dp_size=pc.data_parallel_size, - tp_size=tp_size, - read_mode=get_moriio_mode(kv_transfer_config) == MoRIIOMode.READ, - qp_per_transfer=int(extra_config.get("qp_per_transfer", 1)), - post_batch_size=int(extra_config.get("post_batch_size", -1)), - num_workers=int(extra_config.get("num_workers", 1)), - backend=backend, - transfer_timeout=transfer_timeout, - defer_timeout=defer_timeout, - recv_abort_timeout=recv_abort_timeout, - ) - - -class MoRIIOConstants: - """Constants for MoRIIO connector.""" - - # ZMQ message types - GET_META_MSG = b"get_meta_msg" - POP_DONE_RECV = b"pop_done_recv" - OVER = b"OVER" - COMPLETION_PREFIX = "cmpl" - TRANSFER_PREFIX = "tx" - - PING_INTERVAL = 3 - MAX_PING_RETRIES = 100 - DEFAULT_HANDSHAKE_PORT = "6301" - DEFAULT_NOTIFY_PORT = "61005" - - VLLM_MORI_READ_ABORT_REQUEST_TIMEOUT = 3600 - - # Timeout (seconds) for waiting_for_transfer_complete before raising TransferError. - # Overridable via kv_connector_extra_config["transfer_timeout"]. - DEFAULT_TRANSFER_TIMEOUT = 30.0 - # Timeout (seconds) for deferred WRITE allocation or a missing READ release - # notification. - # Overridable via kv_connector_extra_config["defer_timeout"]. - DEFAULT_DEFER_TIMEOUT = 60.0 - # Timeout (seconds) before an in-flight recv whose RDMA completion was lost - # is aborted, so the decode worker does not hang on it forever. - # Overridable via kv_connector_extra_config["recv_abort_timeout"]. - DEFAULT_RECV_ABORT_TIMEOUT = 120.0 - - -# The router embeds both zmq_addresses in the request_id: -# "___prefill_addr_{zmq}___decode_addr_{zmq}_{32-hex-uuid}" -# MoRIIO zmq_address format: "host:IP,handshake:PORT,notify:PORT" -# -# This lets each connector side parse the peer's connection info without -# requiring the router to pass it explicitly in kv_transfer_params. -_PREFILL_ZMQ_RE = re.compile(r"___prefill_addr_(.+?)___decode_addr_") -# vLLM wraps the router's X-Request-Id as "cmpl---" so there may -# be a trailing "--" suffix after the 32-char UUID. Allow it. -_DECODE_ZMQ_RE = re.compile(r"___decode_addr_(.+)_[0-9a-f]{32}(?:-.*)?$") - - -def parse_moriio_zmq_address( - zmq_address: str, -) -> tuple[str, int, int]: - """Parse the MoRI-IO zmq address into its components. - - Parses ``"host:IP,handshake:PORT,notify:PORT"`` into - (host, handshake_port, notify_port). - - Each key-value pair is split on the *first* colon so that IPv6 addresses - (e.g. ``host:::1``) are handled correctly. Raises ``ValueError`` if any - of ``host``, ``handshake``, or ``notify`` keys are absent or if the port - values are non-numeric. - """ - parts: dict[str, str] = {} - for segment in zmq_address.split(","): - key, _, val = segment.partition(":") - parts[key.strip()] = val.strip() - try: - host = parts["host"] - handshake_port = int(parts["handshake"]) - notify_port = int(parts["notify"]) - except (KeyError, ValueError) as e: - raise ValueError( - f"Malformed zmq_address {zmq_address!r}: expected " - f"'host:IP,handshake:PORT,notify:PORT' format" - ) from e - return host, handshake_port, notify_port - - -def get_peer_zmq_from_request_id(request_id: str, is_producer: bool) -> str | None: - """Extract the *peer's* zmq_address from the vLLM router request_id. - - The producer (prefill) needs the decode's address; the consumer (decode) - needs the prefill's address. - - Returns ``None`` when the request_id does not encode peer info. The - llm-d routing sidecar (``llm-d-inference-scheduler``) does not embed - addresses in ``request_id``; instead it passes ``remote_host``, - ``remote_handshake_port`` and ``remote_notify_port`` explicitly in - ``kv_transfer_params``. Callers must handle the ``None`` return by - falling back to those fields. See ``add_new_req`` for the canonical - fallback path. - """ - if is_producer: - m = _DECODE_ZMQ_RE.search(request_id) - else: - m = _PREFILL_ZMQ_RE.search(request_id) - if m is None: - return None - return m.group(1) - - -@dataclass -class ReqMeta: - """Metadata for a single request.""" - - transfer_id: TransferId - local_block_ids: BlockIds - remote_block_ids: BlockIds - remote_host: str - remote_port: int - remote_handshake_port: int - remote_notify_port: int - remote_engine_id: str - tp_size: int - remote_dp_size: int - # Prefill DP rank that owns this request's KV (forwarded by the proxy). The - # read must target this rank's memory registration; the default 0 preserves - # the symmetric single-DP behaviour. - remote_dp_rank: int = 0 - # Multi-pod: list of remote pod IPs indexed by pod_idx. - multi_pod_hosts: list[str] = field(default_factory=list) - # Per-pod DP size; 0 means fallback to remote_dp_size. - remote_dp_size_local: int = 0 - - -class MoRIIOConnectorMetadata(KVConnectorMetadata): - def __init__(self): - self.reqs_to_recv: dict[ReqId, ReqMeta] = {} - self.reqs_to_save: dict[ReqId, ReqMeta] = {} - self.reqs_to_send: dict[ReqId, float] = {} - self.transfer_id_to_request_id: dict[TransferId, ReqId] = {} - - def __repr__(self): - return ( - f"MoRIIOConnectorMetadata: reqs_to_recv={self.reqs_to_recv}, " - f"reqs_to_save={self.reqs_to_save}, " - f"reqs_to_send={self.reqs_to_send}, " - f"transfer_id_to_request_id={self.transfer_id_to_request_id}" - ) - - def add_new_req( - self, - request_id: ReqId, - local_block_ids: BlockIds, - kv_transfer_params: dict[str, Any], - write_mode=False, - ): - """Ingest a peer's ``kv_transfer_params`` into a typed ``ReqMeta``. - - This is the single place peer info enters the connector. The - ``kv_transfer_params`` contract (as produced by the llm-d sidecar / - vLLM router, or echoed by the prefill leg's ``request_finished``): - - Required (always): - * ``transfer_id`` -- stable id shared by both legs. - * ``remote_engine_id`` -- peer engine id for the handshake table. - * ``remote_block_ids`` -- peer block ids (may be [] in WRITE mode, - where decode allocates its own blocks). - - Peer address -- ONE of the following two must resolve: - * embedded in ``request_id`` (vLLM-router PD id form), OR - * explicit ``remote_host`` + ``remote_handshake_port`` + - ``remote_notify_port`` (llm-d sidecar / returnable path). - If neither resolves we raise -- there is no safe default host/port. - - Optional (defaulted): - * ``tp_size`` (default 1) -- peer TP size. - * ``remote_dp_size`` (default 1) -- peer GLOBAL DP size. - * ``remote_dp_size_local`` (default = ``remote_dp_size``) -- per-pod - DP size for Wide-EP multi-pod port/host - folding; 0/absent means single-pod. - * ``remote_hosts`` (default [remote_host]) -- per-pod IP list - indexed by ``pod_idx = rank // dp_local``. - - Routing keys consumed elsewhere (NOT here): ``remote_dp_rank`` / - ``remote_dp_rank_override`` gate the decode->prefill notify target in - MoRIIOConnectorScheduler; they are router-authoritative and never - self-derived (see that class's request routing contract). - """ - transfer_id = kv_transfer_params["transfer_id"] - - # Try request_id embedded address first, fallback to explicit params. - peer_zmq = get_peer_zmq_from_request_id(request_id, is_producer=write_mode) - if peer_zmq is not None: - remote_host, remote_handshake_port, remote_notify_port = ( - parse_moriio_zmq_address(peer_zmq) - ) - else: - try: - remote_host = kv_transfer_params["remote_host"] - if not remote_host: - raise ValueError( - f"request_id {request_id!r} does not embed a peer " - f"zmq_address and kv_transfer_params['remote_host'] is " - f"empty; cannot route MoRI-IO transfer" - ) - remote_handshake_port = int(kv_transfer_params["remote_handshake_port"]) - remote_notify_port = int(kv_transfer_params["remote_notify_port"]) - except (KeyError, TypeError, ValueError) as e: - raise ValueError( - f"request_id {request_id!r} does not embed a peer " - f"zmq_address and kv_transfer_params is missing one or " - f"more sidecar-fallback keys (need remote_host, " - f"remote_handshake_port, remote_notify_port): {e}" - ) from e - - # Multi-pod: use multi_pod_hosts list or fallback to single host. - _pod_hosts = kv_transfer_params.get("remote_hosts") or [remote_host] - if not isinstance(_pod_hosts, list): - _pod_hosts = [_pod_hosts] - _pod_hosts = [str(h) for h in _pod_hosts] - _remote_dp_size_local = int( - kv_transfer_params.get( - "remote_dp_size_local", - kv_transfer_params.get("remote_dp_size", 1), - ) - ) - - _req = ReqMeta( - transfer_id=transfer_id, - local_block_ids=local_block_ids, - remote_block_ids=kv_transfer_params["remote_block_ids"], - remote_engine_id=kv_transfer_params["remote_engine_id"], - remote_host=remote_host, - remote_port=int(remote_handshake_port), - remote_handshake_port=int(remote_handshake_port), - remote_notify_port=int(remote_notify_port), - # Remote peer TP degree (used as remote_tp_size downstream). The - # decode side must know the prefiller's TP degree to map producer - # ranks onto the correct decode rank (and vice versa). The old plain - # "tp_size" key (#46332) is absent on WRITE producer requests and - # defaulted to 1, collapsing every producer rank onto decode rank 0 - # and corrupting the transfer; prefer "remote_tp_size". 0 == unknown - # (treated as homogeneous downstream). - tp_size=int( - kv_transfer_params.get("remote_tp_size") - or kv_transfer_params.get("tp_size") - or 0 - ), - remote_dp_size=kv_transfer_params.get("remote_dp_size", 1), - remote_dp_rank=kv_transfer_params.get("remote_dp_rank", 0), - multi_pod_hosts=_pod_hosts, - remote_dp_size_local=_remote_dp_size_local, - ) - if write_mode: - self.reqs_to_save[request_id] = _req - else: - self.reqs_to_recv[request_id] = _req - - -@contextlib.contextmanager -def zmq_ctx(socket_type: Any, addr: str) -> Iterator[zmq.Socket]: - """Context manager for a ZMQ socket.""" - if socket_type not in (zmq.ROUTER, zmq.REQ, zmq.DEALER): - raise ValueError(f"Unexpected socket type: {socket_type}") - - ctx: zmq.Context | None = None - try: - ctx = zmq.Context() # type: ignore[attr-defined] - yield make_zmq_socket( - ctx=ctx, path=addr, socket_type=socket_type, bind=socket_type == zmq.ROUTER - ) - finally: - if ctx is not None: - ctx.destroy(linger=0) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_connector.py deleted file mode 100644 index 9760d8bef1..0000000000 --- a/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_connector.py +++ /dev/null @@ -1,3856 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -import logging -import math -import queue -import threading -import time -from collections import OrderedDict, defaultdict -from collections.abc import Collection -from concurrent.futures import Future, ThreadPoolExecutor -from typing import TYPE_CHECKING, Any, cast - -import msgpack -import msgspec -import numpy as np -import torch -import zmq - -from vllm.config import CUDAGraphMode, KVTransferConfig, VllmConfig -from vllm.distributed.kv_transfer.kv_connector.utils import ( - BlockIds, - clip_ssm_state_blocks, -) -from vllm.distributed.kv_transfer.kv_connector.v1.base import ( - KVConnectorBase_V1, - KVConnectorMetadata, - KVConnectorRole, - KVConnectorTransferResults, - SupportsHMA, -) -from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_common import ( - ROLE, - EngineId, - HandshakeError, - MoRIIOAgentMetadata, - MoRIIOConfig, - MoRIIOConnectorMetadata, - MoRIIOConstants, - MoRIIOError, - MoRIIOMode, - MoRIIOTransferAck, - ReqId, - ReqMeta, - TransferBatchState, - TransferError, - TransferId, - WriteTask, - fold_local_rank, - get_moriio_mode, - get_peer_zmq_from_request_id, - get_port_offset, - get_role, - parse_moriio_zmq_address, - pod_index, - resolve_host_ip, - set_role, - zmq_ctx, -) -from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_engine import ( - MoRIIOWrapper, - MoRIIOWriter, -) -from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_heartbeat import ( - MoRIIOHeartbeat, -) -from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_layout import ( - LayerTransferGeometry, - MambaOffsetTemplate, - apply_mamba_offset_template, - build_layer_to_spec, - build_mamba_offset_template, - compute_block_transfer_offsets, - compute_mamba_conv_split_count, - get_layer_transfer_geometry, - is_mla_cache_layer, - iter_layer_registration_regions, - kda_conv_ssm, -) -from vllm.distributed.kv_transfer.kv_connector.v1.ssm_conv_transfer_utils import ( - MambaConvSplitInfo, - derive_mamba_conv_split, -) -from vllm.distributed.parallel_state import ( - get_tensor_model_parallel_world_size, - get_tp_group, - get_world_group, -) -from vllm.forward_context import ForwardContext, get_forward_context -from vllm.logger import init_logger -from vllm.utils.math_utils import cdiv -from vllm.utils.network_utils import ( - make_zmq_path, - make_zmq_socket, -) -from vllm.v1.attention.selector import get_attn_backend -from vllm.v1.core.sched.output import SchedulerOutput -from vllm.v1.kv_cache_interface import ( - KVCacheConfig, - MambaSpec, - SlidingWindowSpec, - is_full_attention_spec, -) -from vllm.v1.metrics.cache_hit_source import CacheHitSource -from vllm.v1.outputs import KVConnectorOutput -from vllm.v1.request import RequestStatus - -if TYPE_CHECKING: - from vllm.v1.attention.backend import AttentionMetadata - from vllm.v1.core.kv_cache_manager import KVCacheBlocks - from vllm.v1.request import Request - -logger = init_logger(__name__) - -# Backoff bounds for re-posting an RDMA READ that the NIC rejected because the -# send queue was full. A separate CQ-poll thread drains completions, so the -# wait only needs to be long enough for it to make progress. -_SQ_FULL_BACKOFF_INITIAL_S = 0.001 -_SQ_FULL_BACKOFF_MAX_S = 0.05 -_MAX_LOCAL_MAMBA_TAIL_BLOCKS = 1 -# A WRITE failure is broadcast to every decode DP rank, so ranks that never -# see the request would otherwise keep its transfer id forever. -_MAX_UNMATCHED_WRITE_FAILURES = 4096 - - -try: - from mori.io import ( - BackendType, - IOEngine, - IOEngineConfig, - ) - - logger.info("MoRIIO is available") - MoRIIO_enabled = True -except ImportError: - logger.error("MoRIIO is not available") - MoRIIO_enabled = False - - -def is_moriio_available() -> bool: - return MoRIIO_enabled - - -def get_moriio_remote_tp_rank( - local_tp_rank: int, local_tp_size: int, remote_tp_size: int -) -> int: - if local_tp_size <= 0 or remote_tp_size <= 0: - raise ValueError("TP sizes must be positive") - if local_tp_rank < 0 or local_tp_rank >= local_tp_size: - raise ValueError( - f"local_tp_rank {local_tp_rank} must be in [0, {local_tp_size})" - ) - if remote_tp_size == local_tp_size: - return local_tp_rank - if remote_tp_size > local_tp_size: - if remote_tp_size % local_tp_size != 0: - raise ValueError( - f"remote tp_size {remote_tp_size} must be a multiple of local " - f"tp_size {local_tp_size} for heterogeneous-TP P/D" - ) - return local_tp_rank * (remote_tp_size // local_tp_size) - if local_tp_size % remote_tp_size != 0: - raise ValueError( - f"local tp_size {local_tp_size} must be a multiple of remote " - f"tp_size {remote_tp_size} for heterogeneous-TP P/D" - ) - return local_tp_rank // (local_tp_size // remote_tp_size) - - -def validate_moriio_heterogeneous_tp_kv_heads( - local_tp_size: int, - remote_tp_size: int, - total_num_kv_heads: int, - is_mla: bool, -) -> None: - if is_mla or local_tp_size == remote_tp_size: - return - if local_tp_size <= 0 or remote_tp_size <= 0 or total_num_kv_heads <= 0: - raise ValueError("TP sizes and total_num_kv_heads must be positive") - if min(local_tp_size, remote_tp_size) >= total_num_kv_heads: - return - raise NotImplementedError( - "MoRIIO heterogeneous TP requires replicated KV heads on both " - f"prefill and decode. Got total_num_kv_heads={total_num_kv_heads}, " - f"local_tp_size={local_tp_size}, remote_tp_size={remote_tp_size}." - ) - - -def get_moriio_expected_ack_count(producer_tp_size: int, consumer_tp_size: int) -> int: - if producer_tp_size <= 0 or consumer_tp_size <= 0: - raise ValueError("TP sizes must be positive") - if consumer_tp_size <= producer_tp_size: - return 1 - if consumer_tp_size % producer_tp_size != 0: - raise ValueError( - f"consumer tp_size {consumer_tp_size} must be a multiple of " - f"producer tp_size {producer_tp_size} for heterogeneous-TP P/D" - ) - return consumer_tp_size // producer_tp_size - - -def resolve_moriio_transfer_ack( - ack: MoRIIOTransferAck | TransferId, - producer_tp_size: int, - live_transfer_ids: Collection[TransferId], - notification_counts: dict[TransferId, int], - completed_transfer_ids: set[TransferId], -) -> TransferId | None: - if isinstance(ack, str): - ack = MoRIIOTransferAck(ack) - transfer_id = ack.transfer_id - if transfer_id not in live_transfer_ids: - return None - if transfer_id in completed_transfer_ids: - return None - - expected_acks = get_moriio_expected_ack_count( - producer_tp_size, ack.consumer_tp_size - ) - count = notification_counts.get(transfer_id, 0) + 1 - if count < expected_acks: - notification_counts[transfer_id] = count - return None - - notification_counts.pop(transfer_id, None) - completed_transfer_ids.add(transfer_id) - return transfer_id - - -class MoRIIOConnector(KVConnectorBase_V1, SupportsHMA): - _cache_hit_source = CacheHitSource.P2P - - @classmethod - def supports_sleep_mode(cls, kv_transfer_config: KVTransferConfig) -> bool: - return False - - @property - def supports_divergent_local_hybrid_hits(self) -> bool: - # The READ path always transfers the recurrent-state slot, including - # when the attention prefix is already present locally. - return self.mode == MoRIIOMode.READ and self._transfers_mamba_state - - def __init__( - self, - vllm_config: VllmConfig, - role: KVConnectorRole, - kv_cache_config: "KVCacheConfig", - ): - super().__init__(vllm_config, role, kv_cache_config) - assert vllm_config.kv_transfer_config is not None, ( - "kv_transfer_config must be set for MoRIIOConnector" - ) - - self.kv_transfer_config = vllm_config.kv_transfer_config - self._set_port_defaults(vllm_config) - - self.engine_id = ( - str(resolve_host_ip(self.kv_transfer_config.kv_connector_extra_config)) - + ":" - + str(self.kv_transfer_config.kv_connector_extra_config["handshake_port"]) - ) - self.mode = get_moriio_mode(self.kv_transfer_config) - self._transfers_mamba_state = any( - group.enable_kv_transfer and isinstance(group.kv_cache_spec, MambaSpec) - for group in kv_cache_config.kv_cache_groups - ) - if role == KVConnectorRole.SCHEDULER: - self.connector_scheduler: MoRIIOConnectorScheduler | None = ( - MoRIIOConnectorScheduler(vllm_config, self.engine_id, kv_cache_config) - ) - self.connector_worker: MoRIIOConnectorWorker | None = None - elif role == KVConnectorRole.WORKER: - self.connector_scheduler = None - self.connector_worker = MoRIIOConnectorWorker( - vllm_config, self.engine_id, kv_cache_config - ) - logger.info( - "Initialized MoRIIO Connector,engine_id:%s,role: %s", - self.engine_id, - role.value, - ) - - ############################################################ - # Scheduler Side Methods - ############################################################ - - def _set_port_defaults(self, vllm_config: VllmConfig): - assert vllm_config.kv_transfer_config is not None, ( - "kv_transfer_config must be set for MoRIIOConnector" - ) - kv_transfer_config = vllm_config.kv_transfer_config - extra_config = kv_transfer_config.kv_connector_extra_config - - if "handshake_port" not in extra_config or not extra_config["handshake_port"]: - extra_config["handshake_port"] = MoRIIOConstants.DEFAULT_HANDSHAKE_PORT - - if "notify_port" not in extra_config or not extra_config["notify_port"]: - extra_config["notify_port"] = MoRIIOConstants.DEFAULT_NOTIFY_PORT - - def get_num_new_matched_tokens( - self, request: "Request", num_computed_tokens: int - ) -> tuple[int, bool]: - assert self.connector_scheduler is not None - return self.connector_scheduler.get_num_new_matched_tokens( - request, num_computed_tokens - ) - - def update_state_after_alloc( - self, request: "Request", blocks: "KVCacheBlocks", num_external_tokens: int - ): - assert self.connector_scheduler is not None - return self.connector_scheduler.update_state_after_alloc( - request, blocks, num_external_tokens, self.connector_worker - ) - - def on_new_request(self, request: "Request") -> None: - """Fired when the scheduler admits a request, before any allocation.""" - assert self.connector_scheduler is not None - self.connector_scheduler.on_new_request(request) - - def build_connector_meta( - self, - scheduler_output: SchedulerOutput, - ) -> KVConnectorMetadata: - assert self.connector_scheduler is not None - return self.connector_scheduler.build_connector_meta(scheduler_output) - - def request_finished( - self, - request: "Request", - block_ids: list[int], - ) -> tuple[bool, dict[str, Any] | None]: - assert self.connector_scheduler is not None - return self.connector_scheduler.request_finished(request, (block_ids,)) - - def request_finished_all_groups( - self, - request: "Request", - block_ids: BlockIds, - ) -> tuple[bool, dict[str, Any] | None]: - assert self.connector_scheduler is not None - if ( - self.connector_scheduler._has_mamba - and self.connector_scheduler.mode == MoRIIOMode.READ - ): - attn_block_ids, mamba_block_groups = ( - self.connector_scheduler.split_block_groups(block_ids) - ) - delay_free, params = self.connector_scheduler.request_finished( - request, attn_block_ids, mamba_block_groups - ) - return delay_free, params - return self.connector_scheduler.request_finished(request, block_ids) - - def update_connector_output(self, connector_output: KVConnectorOutput) -> None: - assert self.connector_scheduler is not None - self.connector_scheduler.update_connector_output(connector_output) - - ############################################################ - # Worker Side Methods - ############################################################ - def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]): - assert self.connector_worker is not None - self.connector_worker.register_kv_caches(kv_caches) - - def get_finished(self, finished_req_ids: set[str]) -> tuple[set[str], set[str]]: - """Get the finished recving and sending requests.""" - assert self.connector_worker is not None - return self.connector_worker.get_finished() - - def get_transfer_results( - self, finished_req_ids: set[str] - ) -> KVConnectorTransferResults: - assert self.connector_worker is not None - return self.connector_worker.get_transfer_results() - - def get_block_ids_with_load_errors(self) -> set[int]: - """Blocks whose MoRIIO read failed, for the scheduler to recompute.""" - if self.connector_worker is None: - return set() - return self.connector_worker.get_block_ids_with_load_errors() - - def start_load_kv(self, forward_context: "ForwardContext", **kwargs) -> None: - assert self.connector_worker is not None - if self.mode == MoRIIOMode.WRITE and get_role() == ROLE.CONSUMER: - self.connector_worker.moriio_wrapper.async_wait_reqid() - - assert isinstance(self._connector_metadata, MoRIIOConnectorMetadata) - self.connector_worker.start_load_kv(self._connector_metadata) - - def wait_for_layer_load(self, layer_name: str) -> None: - assert self.connector_worker is not None - self.connector_worker.wait_for_layer_load(layer_name) - - def save_kv_layer( - self, - layer_name: str, - kv_layer: torch.Tensor, - attn_metadata: "AttentionMetadata", - **kwargs, - ) -> None: - # Only producer/prefill saves KV Cache - if get_role() == ROLE.CONSUMER: - return - assert self.connector_worker is not None, ( - "save_kv_layer called on scheduler role" - ) - - assert isinstance(self._connector_metadata, MoRIIOConnectorMetadata), ( - "Connector metadata not initialized yet" - ) - self.connector_worker.save_kv_layer( - self._connector_metadata, layer_name, kv_layer, attn_metadata, **kwargs - ) - - return None - - def wait_for_save(self): - if self.mode != MoRIIOMode.WRITE or get_role() != ROLE.PRODUCER: - return - assert self.connector_worker is not None - assert isinstance(self._connector_metadata, MoRIIOConnectorMetadata), ( - "Connector metadata not initialized yet" - ) - self.connector_worker.wait_for_save(self._connector_metadata) - - def has_pending_push_work(self) -> bool: - return ( - self.mode == MoRIIOMode.WRITE - and self.connector_scheduler is not None - and bool(self.connector_scheduler._deferred_send_deadlines) - ) - - def get_loaded_kv_cache_group_ids(self, request: "Request") -> tuple[int, ...]: - return self._kv_cache_config.transfer_group_ids - - def shutdown(self): - if self.connector_worker is not None: - self.connector_worker.shutdown() - if self.connector_scheduler is not None: - self.connector_scheduler.shutdown() - - def has_connector_metadata(self) -> bool: - """Check whether the connector metadata is currently set. - - Returns: - bool: True if connector metadata exists, False otherwise. - - """ - try: - return self._connector_metadata is not None - except AttributeError: - return False - - -def _split_kv_cache_group_kinds( - kv_cache_config: "KVCacheConfig", -) -> tuple[list[int], list[int]]: - """Classify kv_cache_groups into (attention_group_indices, - mamba_group_indices) by whether each group's spec is a MambaSpec. - - Groups that opt out of external transfer are absent from - ``KVCacheConfig.transfer_groups``. Returned indices address that filtered - sequence and therefore also the connector's transfer block payload. - """ - attn: list[int] = [] - mamba: list[int] = [] - for gi, group in enumerate(kv_cache_config.transfer_groups): - if isinstance(group.kv_cache_spec, MambaSpec): - mamba.append(gi) - else: - attn.append(gi) - return attn, mamba - - -def _validate_hybrid_speculation(vllm_config: VllmConfig) -> None: - speculative_config = vllm_config.speculative_config - if speculative_config is None: - return - # Other hybrid speculative methods need their own transfer validation. - if not speculative_config.use_dspark(): - raise MoRIIOError( - "MoRIIO hybrid transfer supports DSpark speculative decoding only, got " - f"method={speculative_config.method!r}" - ) - - -def _validate_mamba_specs(specs: Collection[MambaSpec]) -> MambaSpec | None: - """Validate the common packed-state contract and return its representative.""" - specs = tuple(specs) - if not specs: - return None - for spec in specs: - if len(spec.shapes) != 2 or len(spec.dtypes) != 2: - raise MoRIIOError( - "MoRIIO hybrid transfer supports exactly two Mamba states " - "(conv and SSM), got " - f"{len(spec.shapes)} shapes and {len(spec.dtypes)} dtypes" - ) - reference = specs[0] - if any(spec != reference for spec in specs[1:]): - raise MoRIIOError( - "MoRIIO hybrid transfer requires all transferable Mamba groups " - "to use the same cache spec" - ) - return reference - - -class MoRIIOConnectorScheduler: - """Implementation of Scheduler side methods.""" - - def __init__( - self, - vllm_config: VllmConfig, - engine_id: str, - kv_cache_config: "KVCacheConfig", - ): - self.vllm_config = vllm_config - self.kv_cache_config = kv_cache_config - assert vllm_config.kv_transfer_config is not None, ( - "kv_transfer_config must be set for MoRIIOConnector" - ) - self.kv_transfer_config = vllm_config.kv_transfer_config - self.mode = get_moriio_mode(self.kv_transfer_config) - self._attn_group_ids, self._mamba_group_ids = _split_kv_cache_group_kinds( - kv_cache_config - ) - self._has_mamba = bool(self._mamba_group_ids) - mamba_spec = None - if self._has_mamba: - if not self._attn_group_ids: - raise MoRIIOError("MoRIIO hybrid transfer requires an attention group") - if self.mode == MoRIIOMode.READ and len(self._attn_group_ids) != 1: - raise MoRIIOError( - "MoRIIO hybrid READ requires exactly one transferable " - "attention group, got " - f"{len(self._attn_group_ids)}; a drafter that owns a " - "separate attention group is not supported, give it the " - "target's group or disable KV transfer for it" - ) - mamba_specs = [ - cast(MambaSpec, kv_cache_config.transfer_groups[group_id].kv_cache_spec) - for group_id in self._mamba_group_ids - ] - mamba_spec = _validate_mamba_specs(mamba_specs) - _validate_hybrid_speculation(vllm_config) - self._num_ssm_scratch_blocks = ( - mamba_spec.num_speculative_blocks if mamba_spec is not None else 0 - ) - self.block_size = vllm_config.cache_config.block_size - attn_block_size = self.block_size - max_decode_tail_tokens = 1 + vllm_config.num_lookahead_tokens - if self._has_mamba: - attn_spec = kv_cache_config.transfer_groups[ - self._attn_group_ids[0] - ].kv_cache_spec - attn_block_size = attn_spec.block_size - if attn_spec.dcp_sharded: - attn_block_size *= ( - vllm_config.parallel_config.decode_context_parallel_size - ) - # Admission can pad the final prompt token to 1 + num_spec rows; - # allocate_slots reserves drafter lookahead in addition to those. - max_decode_tail_tokens += vllm_config.num_speculative_tokens - self._max_decode_tail_blocks = cdiv(max_decode_tail_tokens, attn_block_size) - self.engine_id: EngineId = engine_id - - self._is_hma_required = ( - not vllm_config.scheduler_config.disable_hybrid_kv_cache_manager - and any( - not is_full_attention_spec(g.kv_cache_spec) - for g in kv_cache_config.transfer_groups - ) - ) - if self._is_hma_required: - # MoRIIO handles sliding-window attention and mamba/KDA groups in - # READ mode. Other hybrid cache specs still fail closed. - unsupported = [ - type(g.kv_cache_spec).__name__ - for g in kv_cache_config.transfer_groups - if not ( - is_full_attention_spec(g.kv_cache_spec) - or isinstance(g.kv_cache_spec, (SlidingWindowSpec, MambaSpec)) - ) - ] - if unsupported: - raise NotImplementedError( - "MoRIIO only supports sliding-window hybrid models (e.g. " - "Gemma) and mamba/KDA groups under the hybrid KV cache " - "manager. Unsupported KV " - f"cache group spec(s): {sorted(set(unsupported))}. Pass " - "--disable-hybrid-kv-cache-manager to run without HMA." - ) - if self.mode == MoRIIOMode.WRITE and any( - isinstance(g.kv_cache_spec, SlidingWindowSpec) - for g in kv_cache_config.transfer_groups - ): - raise NotImplementedError( - "MoRIIO WRITE mode does not support sliding-window " - "cache groups. Use READ mode, or pass " - "--disable-hybrid-kv-cache-manager." - ) - - sw_sizes_tokens: list[tuple[int, int]] = [ - (g.kv_cache_spec.sliding_window, g.kv_cache_spec.block_size) - if isinstance(g.kv_cache_spec, SlidingWindowSpec) - else (0, self.block_size) - for g in kv_cache_config.transfer_groups - ] - # +1 conservatively covers window/block boundary overlap. - self.blocks_per_sw = [ - cdiv(n_tokens, block_size) + 1 if n_tokens else 0 - for n_tokens, block_size in sw_sizes_tokens - ] - # Per-group block size over transfer_groups (matching blocks_per_sw and - # the connector's block payload), used to clamp each full-attention group - # to its prompt length on the WRITE save path. - self._group_block_sizes = [ - getattr(g.kv_cache_spec, "block_size", self.block_size) - for g in kv_cache_config.transfer_groups - ] - # The WRITE-mode prompt-block clamp only covers full-attention groups, so - # MoRIIO + speculative decoding is not yet safe on sliding-window / - # hybrid models. TODO: lift once SWA + spec is validated end-to-end. - if ( - self.mode == MoRIIOMode.WRITE - and vllm_config.speculative_config is not None - and any(bps != 0 for bps in self.blocks_per_sw) - ): - raise NotImplementedError( - "MoRIIO with speculative decoding is not yet supported for " - "sliding-window / hybrid models: the prompt-block clamp is only " - "validated for full-attention KV groups." - ) - # In WRITE mode and chunked prefill, we perform the write after the last chunk. - # Hence we need to track once we are in the last chunk. We do this by computing - # len(block_ids[g]) * block_size[g] for a full attn group g, as hybrid groups - # may be clipped and hence not reflect the full sequence length. Here we store - # g and block_size[g] for later. - self._full_attn_group_idx = 0 - self._full_attn_block_size = self.block_size - for gi, group in enumerate(kv_cache_config.kv_cache_groups): - if is_full_attention_spec(group.kv_cache_spec): - self._full_attn_group_idx = gi - self._full_attn_block_size = getattr( - group.kv_cache_spec, "block_size", self.block_size - ) - break - - self.host_ip = resolve_host_ip( - self.kv_transfer_config.kv_connector_extra_config - ) - self.handshake_port = self.kv_transfer_config.kv_connector_extra_config[ - "handshake_port" - ] - logger.info("Initializing MoRIIO Scheduler engine_id = %s", engine_id) - - self.side_notify_port = self.kv_transfer_config.kv_connector_extra_config[ - "notify_port" - ] - self.tp_size = self.vllm_config.parallel_config.tensor_parallel_size - # Local DP rank (0..dp_local-1) for port arithmetic; fold_local_rank - # handles the external-DP sentinel (dp_local==0) so the arithmetic - # can't ZeroDivisionError at init. - _pc = self.vllm_config.parallel_config - self.dp_rank = fold_local_rank( - _pc.data_parallel_rank, _pc.data_parallel_size_local - ) - # Only first-pod ranks originate notify to avoid duplicates. Guarding - # via pod_index keeps _is_kv_master True for the external-DP sentinel - # (single pod) instead of being silently stuck False. - self._is_kv_master = ( - pod_index(_pc.data_parallel_rank, _pc.data_parallel_size_local) == 0 - ) - # Global DP rank for pinned request ownership check. - self._global_dp_rank = self.vllm_config.parallel_config.data_parallel_rank - self.is_producer = self.kv_transfer_config.kv_role == "kv_producer" - # Requests that need to start recv/send. - # New requests are added by update_state_after_alloc in - # the scheduler. Used to make metadata passed to Worker. - self._reqs_need_recv: dict[ReqId, tuple[Request, BlockIds]] = {} - self._reqs_need_save: dict[ReqId, tuple[Request, BlockIds]] = {} - # Snapshot of kv_transfer_params for chunked prefill recovery. - self._req_kv_params: dict[ReqId, dict] = {} - - # For chunked prefill, we perform layer-wise access within the final chunk. - # TODO: Perform transfer at end chunk. - self._reqs_need_pending_save: dict[ReqId, tuple[Request, BlockIds]] = {} - - if self.is_producer: - set_role(ROLE.PRODUCER) - else: - set_role(ROLE.CONSUMER) - # Reqs to send and their expiration time - self._reqs_need_send: dict[ReqId, float] = {} - # Requests whose block freeing is deferred. READ mode uses the deadline - # as a missing-ACK watchdog; WRITE mode waits for the worker ACK because - # only the worker can mark the transfer terminal before blocks are freed. - # Value: (deadline, transfer_id) for unmapping after completion. - self._deferred_send_deadlines: dict[ReqId, tuple[float, TransferId | None]] = {} - self._defer_timeout = float( - self.kv_transfer_config.kv_connector_extra_config.get( - "defer_timeout", MoRIIOConstants.DEFAULT_DEFER_TIMEOUT - ) - ) - # Buffer for early ACKs that arrive before request_finished. - self._pending_sent_acks: dict[ReqId, float] = {} - self.paths: dict[str, zmq.Socket] = {} - self.transfer_id_to_request_id: dict[TransferId, ReqId] = {} - self.request_id_to_transfer_id: dict[ReqId, TransferId] = {} - - def get_exchange_clipped_blocks( - self, - block_ids: BlockIds, - ) -> BlockIds: - """Clip the block list based on the layer type. - - Behavior: - - sliding window: keep only the last window size blocks - - full attention: keep all blocks - """ - if len(block_ids) == 0: - return [] - if len(block_ids) == len(self.kv_cache_config.kv_cache_groups): - block_ids = self.kv_cache_config.select_transfer_block_ids(block_ids) - if not self._is_hma_required: - return [list(blocks) for blocks in block_ids] - return [ - blocks[-self.blocks_per_sw[i] :] if self.blocks_per_sw[i] > 0 else blocks - for i, blocks in enumerate(block_ids) - ] - - def map_request_id(self, request_id: ReqId, transfer_id: TransferId): - self.transfer_id_to_request_id[transfer_id] = request_id - self.request_id_to_transfer_id[request_id] = transfer_id - - def unmap_request_id( - self, request_id: ReqId, transfer_id: TransferId | None = None - ): - """Unmap request_id/transfer_id. Uses transfer_id for lookup if - exact request_id match fails (handles input_processor mutation).""" - if request_id in self.request_id_to_transfer_id: - tid = self.request_id_to_transfer_id[request_id] - del self.request_id_to_transfer_id[request_id] - if tid in self.transfer_id_to_request_id: - del self.transfer_id_to_request_id[tid] - return - - # Fallback: use transfer_id to find original request_id. - if transfer_id is not None and transfer_id in self.transfer_id_to_request_id: - original_rid = self.transfer_id_to_request_id[transfer_id] - if original_rid != request_id: - logger.debug( - "MoRI-IO unmap via transfer_id: %r -> %r", request_id, original_rid - ) - if original_rid in self.request_id_to_transfer_id: - del self.request_id_to_transfer_id[original_rid] - del self.transfer_id_to_request_id[transfer_id] - return - - logger.warning( - "MoRI-IO unmap MISS: rid=%r transfer_id=%r table_size=%d", - request_id, - transfer_id, - len(self.request_id_to_transfer_id), - ) - - def get_num_new_matched_tokens( - self, - request: "Request", - num_computed_tokens: int, - ) -> tuple[int, bool]: - """For remote prefill, pull all prompt blocks from remote - asynchronously relative to engine execution. - - Args: - request (Request): the request object. - num_computed_tokens (int): the number of locally - computed tokens for this request - Returns: - * the number of tokens that can be loaded from the - external KV cache beyond what is already computed. - * true if the external KV cache tokens will be loaded - asynchronously (between scheduler steps). - - """ - if self.is_producer: - # The P-side prompt truncation for hybrid models happens in - # on_new_request, before the scheduler looks the request up in the - # prefix cache. - return 0, False - - num_prompt_tokens = request.num_prompt_tokens - num_external_tokens = max(num_prompt_tokens - num_computed_tokens, 0) - params = request.kv_transfer_params or {} - if not params.get("do_remote_prefill"): - return 0, False - if self.mode == MoRIIOMode.WRITE: - if self._has_mamba: - # P pushes h(N-1). Do not publish it under the N-token hash - # when the scheduler caches a completed asynchronous load. - num_external_tokens = max(num_external_tokens - 1, 0) - return num_external_tokens, num_external_tokens > 0 - - # READ mode always recomputes the last token locally on the decoder. - # - # The second element declares who waits for the KV, not whether it has - # arrived. False keeps the wait on this side: the forward runs in the - # same step, and the connector blocks in wait_for_layer_load (or in - # start_load_kv under full-graph capture) until the KV is really there. - return max(num_external_tokens - 1, 0), False - - def on_new_request(self, request: "Request") -> None: - """Scheduler hook fired when a request is admitted. - - The hybrid P-side truncation must land here rather than in - ``get_num_new_matched_tokens``: the scheduler runs the local prefix - cache lookup first and only then asks the connector, so truncating - there shortens a prompt whose cache hit was already measured against - the full length. On a full hit that leaves ``num_computed_tokens == - num_tokens`` and trips the scheduler's ``num_new_tokens > 0`` assert. - """ - if not (self.is_producer and self._has_mamba): - return - params = request.kv_transfer_params - if params is not None and params.get("do_remote_decode"): - self._truncate_mamba_request_for_prefill(request) - - def _truncate_mamba_request_for_prefill(self, request: "Request") -> None: - """P-side only: drop the last prompt token so the prefiller computes - h(N-1) instead of h(N). - - The decoder recomputes the last token locally to derive h(N) correctly - (see the READ len-1 accounting in get_num_new_matched_tokens). Guarded - by ``_p_side_truncated`` to avoid repeated truncation across a - preemption/reschedule. Mirrors the NIXL scheduler. - """ - params = request.kv_transfer_params - if ( - params is not None - and not params.get("_p_side_truncated") - and request.num_prompt_tokens > 1 - ): - if request.prompt_token_ids is not None: - request.prompt_token_ids.pop() - elif request.prompt_embeds is not None: - request.prompt_embeds = request.prompt_embeds[:-1] - else: - return - - request._all_token_ids.pop() - request.num_prompt_tokens -= 1 - request.max_tokens = 1 - params["_p_side_truncated"] = True - - def send_notify_block( - self, - req_id: ReqId, - transfer_id: TransferId, - block_notify_list: list, - host=None, - port=None, - ): - path = make_zmq_path("tcp", host, port) - if path not in self.paths: - ctx = zmq.Context.instance() - sock = make_zmq_socket( - ctx=ctx, path=path, socket_type=zmq.DEALER, bind=False - ) - self.paths[path] = sock - - data = { - "req_id": req_id, - "transfer_id": transfer_id, - "block_notify_list": block_notify_list or [], - # GLOBAL decode dp rank: producer derives the per-pod notify offset - # (% dp_local), owning pod index (// dp_local), and write-target - # from it. Sending the LOCAL rank made child-pod consumers look - # like master rank 0 and hang in WAITING_FOR_REMOTE_KVS. Single-pod: - # local == global. - "decode_rank": self._global_dp_rank, - "type": "remote_blocks", - } - serialized_data = msgpack.dumps(data) - self.paths[path].send(serialized_data) - - def _send_transfer_release(self, transfer_id: TransferId, host: str, port: int): - path = make_zmq_path("tcp", host, port) - if path not in self.paths: - ctx = zmq.Context.instance() - sock = make_zmq_socket( - ctx=ctx, path=path, socket_type=zmq.DEALER, bind=False - ) - self.paths[path] = sock - - # Advertise the consumer (decode) TP size so prefill counts the right - # number of ACKs (get_moriio_expected_ack_count; see the upstream - # READ-mode release in _pop_done_transfers). Resolves to 1 for our - # homogeneous TP=1 configs; only matters under heterogeneous-TP fan-in. - self.paths[path].send( - msgpack.dumps( - { - "type": "release", - "transfer_id": transfer_id, - "consumer_tp_size": self.tp_size, - } - ) - ) - - def _release_write_prefill_blocks(self, request_id: ReqId, params: dict[str, Any]): - transfer_id = params.get("transfer_id") - if transfer_id is None: - logger.warning( - "Cannot release WRITE prefill blocks for request %s: " - "missing transfer_id", - request_id, - ) - return - - remote_dp_rank = params.get("remote_dp_rank", 0) - remote_host = params.get("remote_host") - remote_notify_port = params.get("remote_notify_port") - if remote_host is None or remote_notify_port is None: - try: - peer_zmq = get_peer_zmq_from_request_id(request_id, is_producer=False) - if peer_zmq is None: - raise ValueError("no peer zmq address for request") - remote_host, _, remote_notify_port = parse_moriio_zmq_address(peer_zmq) - except ValueError: - logger.warning( - "Cannot release WRITE prefill blocks for request %s: " - "missing remote notify address", - request_id, - ) - return - - remote_notify_port = int(remote_notify_port) - for tp_index in range(self.tp_size): - target_port = remote_notify_port + get_port_offset(remote_dp_rank, tp_index) - self._send_transfer_release(transfer_id, remote_host, target_port) - - def update_state_after_alloc( - self, - request: "Request", - blocks: "KVCacheBlocks", - num_external_tokens: int, - connector_worker: "MoRIIOConnectorWorker | None" = None, - ): - """Scheduler-side post-allocation hook (decode leg in WRITE mode). - - In WRITE mode this fires the decode->prefill "blocks ready" notify - that lets the producer RDMA-Write its KV into the freshly allocated - decode blocks. - - DP-rank routing contract (router-authoritative) - ------------------------------------------------ - Both legs of a disagg pair must agree on a single prefill DP rank, - otherwise the notify lands on a rank that never handshook and the - request hangs until ``VLLM_MORIIO_DEFERRED_TIMEOUT_S``. - - The routing authority is the external router/sidecar, NOT the - connector. The llm-d routing sidecar computes ``H = pickDPRank(uuid, - dp_size)`` (see ``dp_rank.go``) once and pins BOTH legs to ``H`` two - ways: the ``X-data-parallel-rank`` dispatch header (which vLLM engine - the leg lands on) and ``kv_transfer_params.remote_dp_rank=H`` (which - prefill rank this notify targets), with ``remote_dp_rank_override=True``. - - The connector consumes ``remote_dp_rank`` verbatim and does NOT - self-derive a rank: an independent hash here could disagree with the - router's dispatch pin and misroute the notify. For the returnable paths - (READ / serial WRITE) the prefill leg also echoes back the rank it ran - on via ``request_finished`` so routing is pure propagation. - - The owning rank is the only one that originates the notify - (``remote_dp_rank_override`` -> global-rank match; otherwise the - ``_is_kv_master`` anti-duplicate gate). See the inline comments - below for the exactly-once-across-pods reasoning. - """ - params = request.kv_transfer_params - if not params: - return - # LLM-D sidecar compat: the nixlv2 routing sidecar emits NIXL-shaped - # kv_transfer_params without MoRI-IO's transfer_id. Synthesize one - # deterministically from request_id so producer and consumer agree - # without a sidecar wire-protocol change. - transfer_id = params.get("transfer_id") or f"sidecar-{request.request_id}" - params.setdefault("transfer_id", transfer_id) - request_id = request.request_id - self.map_request_id(request_id, transfer_id) - if params.get("do_remote_decode") and self.mode == MoRIIOMode.WRITE: - local_block_ids: BlockIds = blocks.get_block_ids() - self._reqs_need_save[request.request_id] = (request, local_block_ids) - # Snapshot params now so chunked-prefill build_connector_meta - # can recover them on the final chunk even if the live - # request.kv_transfer_params has been mutated/cleared. - self._req_kv_params[request.request_id] = dict(params) - - if params is not None and params.get("do_remote_prefill"): - if self.mode == MoRIIOMode.READ: - if remote_block_ids := params.get("remote_block_ids"): - # remote_engine_id is returned by the prefill's request_finished. - # host/ports come from the request_id (parsed in add_new_req). - if "remote_engine_id" in params: - adjusted_remote_block_ids = remote_block_ids - if num_external_tokens > 0: - if self._has_mamba: - # READ is synchronous, so allocate_slots may hash - # destination blocks before this hook runs. Pair - # the attention and recurrent-state suffixes - # independently. - remote_attn = list(remote_block_ids[0]) - remote_mamba_groups = [ - list(group) for group in remote_block_ids[1:] - ] - attn_block_ids, mamba_block_groups = ( - self.split_block_groups(blocks.get_block_ids()) - ) - if len(mamba_block_groups) != len(remote_mamba_groups): - raise MoRIIOError( - "MoRIIO hybrid READ Mamba group " - "count mismatch: " - f"local={len(mamba_block_groups)}, " - f"remote={len(remote_mamba_groups)}" - ) - local_attn = attn_block_ids - local_attn, remote_attn = self._align_read_blocks( - local_attn, - remote_attn, - self._max_decode_tail_blocks, - ) - aligned_local_mamba: list[list[int]] = [] - aligned_remote_mamba: list[list[int]] = [] - # Scratch slots were removed by - # split_block_groups, so only one local running - # state tail can remain. - for local_group, remote_group in zip( - mamba_block_groups, - remote_mamba_groups, - strict=True, - ): - aligned_local, aligned_remote = ( - self._align_read_blocks( - local_group, - remote_group, - _MAX_LOCAL_MAMBA_TAIL_BLOCKS, - ) - ) - aligned_local_mamba.append(aligned_local) - aligned_remote_mamba.append(aligned_remote) - local_block_ids = [local_attn, *aligned_local_mamba] - adjusted_remote_block_ids = [ - remote_attn, - *aligned_remote_mamba, - ] - else: - # Preserve main's per-group HMA representation - # for attention-only hybrid models while - # accounting for the locally recomputed tail. - local_block_ids = self.get_exchange_clipped_blocks( - blocks.get_block_ids() - ) - adjusted_remote_block_ids = [] - aligned_local_block_ids: list[list[int]] = [] - for local_group, remote_group in zip( - local_block_ids, - remote_block_ids, - strict=True, - ): - aligned_local, aligned_remote = ( - self._align_read_blocks( - local_group, - remote_group, - self._max_decode_tail_blocks, - ) - ) - aligned_local_block_ids.append(aligned_local) - adjusted_remote_block_ids.append(aligned_remote) - local_block_ids = aligned_local_block_ids - else: - if self._has_mamba: - # Attention can be a complete local hit, but the - # recurrent state is never prefix-cacheable. - _, mamba_block_groups = self.split_block_groups( - blocks.get_block_ids() - ) - local_block_ids = [[], *mamba_block_groups] - else: - # Nothing to pull, but retain one empty list per - # cache group so the worker can notify P. - local_block_ids = [[] for _ in remote_block_ids] - - self._reqs_need_recv[request.request_id] = ( - request, - local_block_ids, - ) - # Snapshot params for chunked prefill consumption - # in build_connector_meta (see comment above). - saved_params = dict(params) - if num_external_tokens > 0: - saved_params["remote_block_ids"] = adjusted_remote_block_ids - self._req_kv_params[request.request_id] = saved_params - else: - logger.warning( - "Got invalid KVTransferParams: %s. This " - "request will not utilize KVTransfer", - params, - ) - - else: - # WRITE mode, decode side: notify P that blocks are ready - assert request.kv_transfer_params is not None, ( - "kv_transfer_params should not be None" - ) - - remote_dp_rank = request.kv_transfer_params.get("remote_dp_rank", 0) - - # Effective DP fan-out for the per-pod port/host math below - # (see _remote_dp_rank_for_port / _pod_idx). Capped to the - # per-pod local size when the router advertises it (Wide-EP). - _dp_size = int(request.kv_transfer_params.get("remote_dp_size", 1) or 1) - try: - _dp_local = int( - request.kv_transfer_params.get("remote_dp_size_local", 0) or 0 - ) - if _dp_local > 0: - _dp_size = min(_dp_size, _dp_local) - except (TypeError, ValueError): - _dp_local = 0 - - # Rank routing is ROUTER-AUTHORITATIVE: honor the router-pinned - # remote_dp_rank (matched to the X-data-parallel-rank dispatch - # pin). Self-deriving a rank could disagree and notify a rank - # that never served the request. If unpinned in a multi-rank - # deployment, warn instead of guessing. - if ( - _dp_size > 1 - and "remote_dp_rank" not in request.kv_transfer_params - and "is_request_leader" not in request.kv_transfer_params - ): - logger.warning( - "MoRI-IO decode notify: remote_dp_size=%d but the router " - "did not pin remote_dp_rank for request %s; defaulting to " - "rank 0. The router must set remote_dp_rank (and the " - "X-data-parallel-rank dispatch header) so the prefill and " - "decode legs agree on the same rank.", - _dp_size, - request.request_id, - ) - - # Only the owning rank originates notify (exactly-once). - # Priority: is_request_leader > remote_dp_rank_override > _is_kv_master - _leader_flag = request.kv_transfer_params.get("is_request_leader") - if _leader_flag is not None: - _should_notify = bool(_leader_flag) - elif request.kv_transfer_params.get("remote_dp_rank_override"): - _should_notify = self._global_dp_rank == remote_dp_rank - else: - _should_notify = self._is_kv_master - if _should_notify and num_external_tokens == 0: - # Nothing left to fetch (full local prefix hit), so release - # the producer's blocks instead of sending an empty - # allocation, which it rejects. - self._release_write_prefill_blocks( - request.request_id, request.kv_transfer_params - ) - elif _should_notify: - peer_zmq = get_peer_zmq_from_request_id( - request.request_id, is_producer=False - ) - if peer_zmq is not None: - remote_host, _, remote_notify_port = parse_moriio_zmq_address( - peer_zmq - ) - else: - # Sidecar fallback: use explicit params fields. - params = request.kv_transfer_params or {} - remote_host = params.get("remote_host") or "" - try: - remote_notify_port = int( - params.get("remote_notify_port") or 0 - ) - except (TypeError, ValueError): - remote_notify_port = 0 - if not remote_host or not remote_notify_port: - raise ValueError( - f"request {request.request_id!r}: " - f"request_id has no embedded peer " - f"zmq_address and kv_transfer_params is " - f"missing remote_host / remote_notify_port " - f"(got remote_host={remote_host!r}, " - f"remote_notify_port={remote_notify_port!r})" - ) - - block_notify_list: list[int] | list[list[int]] - if self._has_mamba: - block_notify_list = list( - self._clamp_to_prompt_blocks( - request, blocks.get_block_ids() - ) - ) - else: - block_notify_list = blocks.get_block_ids()[0] - - # Wide-EP multi-pod: a pod binds notify sockets only for - # its LOCAL ranks, so the port offset must use the per-pod - # local rank (% dp_local), not the global rank. Single-pod - # is bit-identical (modulus is a no-op). - _remote_dp_rank_for_port = fold_local_rank( - remote_dp_rank, _dp_local - ) - # The target rank may live on a child pod at a different IP, - # so resolve the per-pod host (pod_idx = global // dp_local). - # Otherwise a notify for child ranks lands on the master - # pod and the request hangs in WAITING_FOR_REMOTE_KVS. - _notify_host = remote_host - _kvp = request.kv_transfer_params or {} - _remote_hosts = _kvp.get("remote_hosts") or [] - if _dp_local > 0 and _remote_hosts: - _pod_idx = pod_index(remote_dp_rank, _dp_local) - if 0 <= _pod_idx < len(_remote_hosts): - _notify_host = _remote_hosts[_pod_idx] - for tp_index in range(self.tp_size): - target_port = remote_notify_port + get_port_offset( - _remote_dp_rank_for_port, tp_index - ) - - self.send_notify_block( - req_id=request.request_id, - transfer_id=request.kv_transfer_params["transfer_id"], - block_notify_list=block_notify_list, - host=_notify_host, - port=target_port, - ) - - # Only trigger 1 KV transfer per request. - - params["do_remote_prefill"] = False - - def _clamp_to_prompt_blocks(self, req: "Request", block_ids: BlockIds) -> BlockIds: - """Drop trailing speculative-lookahead blocks the consumer never - allocated, keeping each full-attention group's prompt blocks. - - Args: - req: Request being saved. - block_ids: Per-KV-cache-group local block ids. - - Returns: - ``block_ids`` with every full-attention group truncated to - ``ceil(num_prompt_tokens / block_size)`` leading blocks; - sliding-window groups are left unchanged. - - """ - if self._has_mamba: - # WRITE preserves transfer-group order, including a separate drafter - # attention group. READ retains its attention-first wire format. - transfer_blocks = self.kv_cache_config.select_transfer_block_ids(block_ids) - dcp_size = self.vllm_config.parallel_config.decode_context_parallel_size - clamped_hybrid: list[list[int]] = [] - for group, blocks in zip( - self.kv_cache_config.transfer_groups, transfer_blocks, strict=True - ): - spec = group.kv_cache_spec - if isinstance(spec, MambaSpec): - clamped_hybrid.append( - clip_ssm_state_blocks(list(blocks), spec.num_speculative_blocks) - ) - else: - block_size = spec.block_size * (dcp_size if spec.dcp_sharded else 1) - clamped_hybrid.append( - list(blocks[: cdiv(req.num_prompt_tokens, block_size)]) - ) - return clamped_hybrid - clamped = list(block_ids) - changed = False - for g, group_blocks in enumerate(block_ids): - if self.blocks_per_sw[g] != 0: - continue - num_prompt_blocks = math.ceil( - req.num_prompt_tokens / self._group_block_sizes[g] - ) - if len(group_blocks) > num_prompt_blocks: - clamped[g] = group_blocks[:num_prompt_blocks] - changed = True - return clamped if changed else block_ids - - def _is_final_prefill_chunk( - self, req: "Request", scheduler_output: SchedulerOutput - ) -> bool: - """Whether this step completes the prompt. - - ``build_connector_meta`` runs before ``_update_after_schedule``, so - ``num_computed_tokens`` excludes the current chunk; add this step's - scheduled tokens. Token progress is lookahead-safe unlike block counts. - """ - num_scheduled = scheduler_output.num_scheduled_tokens.get(req.request_id, 0) - return req.num_computed_tokens + num_scheduled >= req.num_prompt_tokens - - def _get_write_blocks( - self, req: "Request", block_ids: BlockIds, scheduler_output: SchedulerOutput - ) -> BlockIds: - if self._has_mamba: - # Align-mode state slots can move without appending a new block. - # Resolve the authoritative table at the final prefill step. - block_state = scheduler_output.kv_connector_block_state - assert block_state is not None - current_blocks = block_state.get_block_ids(req.request_id) - assert current_blocks is not None - block_ids = current_blocks - return self._clamp_to_prompt_blocks(req, block_ids) - - def build_connector_meta( - self, - scheduler_output: SchedulerOutput, - ) -> KVConnectorMetadata: - meta = MoRIIOConnectorMetadata() - meta.transfer_id_to_request_id = self.transfer_id_to_request_id - - if self.mode == MoRIIOMode.WRITE and get_role() == ROLE.PRODUCER: - # This is the logic for checking against chunked prefill. - # When the last chunk is identified, - # It places the request metadata into the saving queue. - - for i, req_id in enumerate(scheduler_output.scheduled_cached_reqs.req_ids): - # Non-disagg requests aren't registered here; skip them - # instead of KeyError-ing. - if req_id not in self._reqs_need_pending_save: - continue - req, existing_blocks = self._reqs_need_pending_save[req_id] - # The final chunk may allocate no new blocks (the previous chunk - # reserved speculative-lookahead slack), so new_block_ids is None; - # still fall through to the final-chunk check below. - new_block_ids = scheduler_output.scheduled_cached_reqs.new_block_ids[i] - if new_block_ids is not None and not self._has_mamba: - existing_blocks = [ - old + new - for old, new in zip(existing_blocks, new_block_ids, strict=True) - ] - self._reqs_need_pending_save[req_id] = (req, existing_blocks) - if not self._is_final_prefill_chunk(req, scheduler_output): - continue - # Live kv_transfer_params may be cleared; prefer the - # update_state_after_alloc snapshot. - kv_params = self._req_kv_params.pop( - req_id, req.kv_transfer_params or {} - ) - save_block_ids = self._get_write_blocks( - req, existing_blocks, scheduler_output - ) - meta.add_new_req( - request_id=req_id, - local_block_ids=save_block_ids, - kv_transfer_params=kv_params, - write_mode=True, - ) - del self._reqs_need_pending_save[req_id] - - # Loop through scheduled reqs and convert to ReqMeta. - for req_id, (req, block_ids) in self._reqs_need_recv.items(): - kv_params = self._req_kv_params.get(req_id, req.kv_transfer_params or {}) - meta.add_new_req( - request_id=req_id, - local_block_ids=block_ids, - kv_transfer_params=kv_params, - ) - if ( - self.mode == MoRIIOMode.READ - and self._has_mamba - and scheduler_output.new_block_ids_to_zero - ): - # Hybrid models zero recycled attention pages that held Mamba state. - # Host-submitted READs overwrite these pages and can race zeroing. - # Hybrid READ metadata puts the aligned attention pages first. - read_dst_block_ids = { - b - for _, block_ids in self._reqs_need_recv.values() - for b in block_ids[0] - } - scheduler_output.new_block_ids_to_zero = [ - b - for b in scheduler_output.new_block_ids_to_zero - if b not in read_dst_block_ids - ] - - for req_id, (req, block_ids) in self._reqs_need_save.items(): - kv_params = self._req_kv_params.get(req_id, req.kv_transfer_params or {}) - if not self._is_final_prefill_chunk(req, scheduler_output): - self._reqs_need_pending_save[req_id] = (req, block_ids) - continue - save_block_ids = self._get_write_blocks(req, block_ids, scheduler_output) - meta.add_new_req( - request_id=req_id, - local_block_ids=save_block_ids, - kv_transfer_params=kv_params, - write_mode=True, - ) - # Clear the list once workers start the transfers - - meta.reqs_to_send = self._reqs_need_send - - # Reclaim snapshot cache entries that completed this step. Keep - # entries that are still pending (chunked prefill not yet at the - # final chunk) — those will be popped above on the final chunk. - for req_id in self._reqs_need_recv: - self._req_kv_params.pop(req_id, None) - for req_id in self._reqs_need_save: - if req_id not in self._reqs_need_pending_save: - self._req_kv_params.pop(req_id, None) - - self._reqs_need_recv.clear() - self._reqs_need_save.clear() - self._reqs_need_send = {} - - return meta - - def shutdown(self): - for path, sock in self.paths.items(): - try: - sock.close(linger=0) - logger.debug("Closed ZMQ socket for path: %s", path) - except Exception as e: - logger.warning("Error closing ZMQ socket for path %s: %s", path, e) - self.paths.clear() - - def split_block_groups( - self, block_ids: list[list[int]] | tuple[list[int], ...] - ) -> tuple[list[int], list[list[int]]]: - """Select transferable attention and Mamba block groups. - - The wire payload stores the attention group first, followed by every - Mamba group in transfer-group order. Only each group's running-state - slot is transferred. - """ - if not block_ids: - return [], [] - if not self._has_mamba: - raise MoRIIOError("Mamba block groups requested for a non-hybrid model") - transfer_block_ids = self.kv_cache_config.select_transfer_block_ids(block_ids) - attn = list(transfer_block_ids[self._attn_group_ids[0]]) - mamba_groups = [ - clip_ssm_state_blocks( - list(transfer_block_ids[group_id]), - self._num_ssm_scratch_blocks, - ) - for group_id in self._mamba_group_ids - ] - return attn, mamba_groups - - @staticmethod - def _align_read_blocks( - local_block_ids: list[int], - remote_block_ids: list[int], - max_decode_tail_blocks: int, - ) -> tuple[list[int], list[int]]: - """Pair a decode suffix with the corresponding prefill blocks. - - Prefix hits can make the decode's unhashed suffix shorter than the - prefill's full list, so read from the matching remote suffix. The - decoder can instead have extra tail blocks for the locally recomputed - final token and speculative slots; these have no remote source and are - not transferred. - """ - if len(local_block_ids) > len(remote_block_ids): - extra_decode_blocks = len(local_block_ids) - len(remote_block_ids) - if extra_decode_blocks > max_decode_tail_blocks: - raise ValueError( - "decode block ids exceed prefill block ids beyond the " - f"allowed local tail ({max_decode_tail_blocks} blocks): " - f"{len(local_block_ids)} > {len(remote_block_ids)}" - ) - local_block_ids = local_block_ids[: len(remote_block_ids)] - elif len(local_block_ids) < len(remote_block_ids): - remote_block_ids = ( - remote_block_ids[-len(local_block_ids) :] if local_block_ids else [] - ) - return local_block_ids, remote_block_ids - - def request_finished( - self, - request: "Request", - block_ids: BlockIds | list[int], - mamba_block_groups: list[list[int]] | None = None, - ) -> tuple[bool, dict[str, Any] | None]: - """Once a request is finished, determine whether request blocks - should be freed now or will be sent asynchronously and freed later. - - ``block_ids`` normally contains all cache groups. Hybrid READ callers from - ``request_finished_all_groups`` pass a flat attention list and the - recurrent-state groups separately in ``mamba_block_groups``. - """ - request_id = request.request_id - params = request.kv_transfer_params - # Consumer: can unmap transfer_id<->request_id immediately since done_recving - # has fired at this point (i.e. KV has been transferred) - # Producer: must keep the mapping until we get notification that blocks can - # be freed, which may be several scheduler steps later. - if not self.is_producer: - transfer_id = params.get("transfer_id") if params else None - self.unmap_request_id(request_id, transfer_id=transfer_id) - logger.debug( - "MoriioConnector request_finished, request_status=%s, " - "kv_transfer_params=%s", - request.status, - params, - ) - if not params: - return False, None - - if params.get("do_remote_prefill"): - # If do_remote_prefill is still True when the request is finished, - # update_state_after_alloc must not have been called (the request - # must have been aborted before it was scheduled). - # To avoid stranding the prefill blocks in the prefill instance, - # READ mode adds empty block_ids to _reqs_need_recv so the worker - # side notifies the prefill instance. WRITE mode should notify the - # producer directly: there is no decode allocation for the producer - # to write into, and a plain request_id may not contain router- - # embedded MoRIIO ZMQ addresses. - if self.mode == MoRIIOMode.WRITE: - self._release_write_prefill_blocks(request.request_id, params) - else: - # Carry the actually-allocated blocks (incl. the per-request KDA - # mamba slot) instead of []: a do_remote_prefill request finishing - # before update_state_after_alloc still has blocks, and dropping the - # mamba slot here would re-trip the KDA guard in _read_blocks. - if mamba_block_groups: - recv_blocks: BlockIds = [ - list(cast(list[int], block_ids)), - *[list(group) for group in mamba_block_groups], - ] - elif block_ids and isinstance(block_ids[0], int): - recv_blocks = [list(cast(list[int], block_ids))] - else: - grouped_block_ids = cast(BlockIds, block_ids) - recv_blocks = ( - [list(group) for group in grouped_block_ids] - if self._has_mamba - else [[] for _ in self.blocks_per_sw] - ) - self._reqs_need_recv[request.request_id] = (request, recv_blocks) - params["do_remote_prefill"] = False - return False, None - - if ( - not params.get("do_remote_decode") - or request.status != RequestStatus.FINISHED_LENGTH_CAPPED - ): - return False, None - - # On producer - if mamba_block_groups is not None: - computed_block_ids: BlockIds = [ - list(cast(list[int], block_ids)), - *[list(group) for group in mamba_block_groups], - ] - elif self._has_mamba and self.mode == MoRIIOMode.WRITE: - computed_block_ids = self._clamp_to_prompt_blocks( - request, cast(BlockIds, block_ids) - ) - elif self._has_mamba: - raise MoRIIOError("Hybrid request completion requires all KV cache groups") - else: - computed_block_ids = self.get_exchange_clipped_blocks( - cast(BlockIds, block_ids) - ) - delay_free_blocks = any(len(group) > 0 for group in computed_block_ids) - - if delay_free_blocks: - # Prefill request on remote. It will be read from D upon completion - self._reqs_need_send[request.request_id] = ( - time.perf_counter() - + MoRIIOConstants.VLLM_MORI_READ_ABORT_REQUEST_TIMEOUT - ) - self._deferred_send_deadlines[request.request_id] = ( - time.monotonic() + self._defer_timeout, - params.get("transfer_id") if params else None, - ) - - # Return KV transfer params forwarded verbatim to the decode instance by - # the router. remote_dp_rank is the rank THIS prefill leg actually ran - # on: on the returnable paths (READ / serial WRITE) the router forwards - # it to the decode leg, so the decode->prefill notify is routed by pure - # propagation of the producer's real rank -- no hashing, no reliance on - # the router independently pinning the same rank. remote_dp_rank_override - # makes the decode side honor it via the global-rank match gate. - return delay_free_blocks, dict( - do_remote_prefill=True, - do_remote_decode=False, - remote_block_ids=computed_block_ids, - remote_engine_id=self.engine_id, - remote_host=self.host_ip, - remote_handshake_port=self.handshake_port, - remote_notify_port=self.side_notify_port, - remote_dp_rank=self._global_dp_rank, - remote_dp_rank_override=True, - remote_dp_size=self.vllm_config.parallel_config.data_parallel_size, - remote_dp_size_local=( - self.vllm_config.parallel_config.data_parallel_size_local - ), - tp_size=self.vllm_config.parallel_config.tensor_parallel_size, - transfer_id=params["transfer_id"], - ) - - def update_connector_output(self, connector_output: KVConnectorOutput) -> None: - """Reconcile worker finished_sending ACKs with the producer lifecycle. - - Called every scheduler step (when there is KV connector output), - BEFORE the scheduler's finished_sending free loop. We rewrite - ``connector_output.finished_sending`` in place so it contains - exactly the producer requests that are safe for the scheduler to - free this step, letting the scheduler keep the plain upstream - ``assert req_id in self.requests`` + ``_free_blocks`` path: - - * An ACK whose request is in ``_deferred_send_deadlines`` - (request_finished ran with delay_free_blocks=True, so the - scheduler is holding its blocks) is surfaced -> freed now. - * An ACK that arrives BEFORE its request finished is parked in - ``_pending_sent_acks`` and released on a later step once the - request enters ``_deferred_send_deadlines``. - * In READ mode, a deferred send whose ACK never arrives is reaped - after ``_defer_timeout``. WRITE-mode timeouts are owned by the worker, - which marks the transfer terminal before ACKing the block release. - * A parked ACK that never matches a deferral before its own - deadline is a stale duplicate (e.g. a real ACK landing after the - send was already reaped) and is dropped. - - Consumers never populate finished_sending (they report - finished_recving), and they unmap in request_finished, so this is a - no-op for them. - """ - if not self.is_producer: - return - - incoming = set(connector_output.finished_sending or ()) - now = time.monotonic() - - # Surface ACKs whose request is already finished (blocks held for - # delayed free); park the rest until their request finishes. - safe: set[ReqId] = set() - for req_id in incoming: - if req_id in self._deferred_send_deadlines: - safe.add(req_id) - else: - self._pending_sent_acks.setdefault(req_id, now + self._defer_timeout) - - # Release previously parked ACKs whose request has since finished. - for req_id in self._pending_sent_acks: - if req_id in self._deferred_send_deadlines: - safe.add(req_id) - - # Reap deferred sends whose ACK never arrived (avoid leaking blocks). - expired = [ - req_id - for req_id, (deadline, _) in self._deferred_send_deadlines.items() - if self.mode == MoRIIOMode.READ and now >= deadline - ] - if expired: - safe.update(expired) - logger.warning( - "Reaped %d deferred sends with no finished_sending " - "notification after %.0fs. This indicates lost async KV " - "completion notifications from the KV connector.", - len(expired), - self._defer_timeout, - ) - - # Finalize the requests we are surfacing: drop their deferral/park - # bookkeeping and unmap their transfer ids. - for req_id in safe: - deferred_info = self._deferred_send_deadlines.pop(req_id, None) - transfer_id = deferred_info[1] if deferred_info else None - self._pending_sent_acks.pop(req_id, None) - self.unmap_request_id(req_id, transfer_id=transfer_id) - - # Drop stale parked ACKs that never matched a deferral in time. - stale = [ - req_id - for req_id, deadline in self._pending_sent_acks.items() - if now >= deadline - ] - for req_id in stale: - self._pending_sent_acks.pop(req_id, None) - - connector_output.finished_sending = safe or None - - -class MoRIIOConnectorWorker: - """Implementation of Worker side methods.""" - - def __init__( - self, - vllm_config: VllmConfig, - engine_id: str, - kv_cache_config: "KVCacheConfig", - ): - if not is_moriio_available(): - raise RuntimeError( - "MoRIIO is not available. Please ensure the 'mori' package " - "is installed and properly configured." - ) - - assert vllm_config.kv_transfer_config is not None - self.moriio_config = MoRIIOConfig.from_vllm_config(vllm_config) - self.mode = ( - MoRIIOMode.READ if self.moriio_config.read_mode else MoRIIOMode.WRITE - ) - - logger.info("Initializing MoRIIO worker %s", engine_id) - - logging.getLogger("aiter").disabled = True - - # Config. - self.vllm_config = vllm_config - assert vllm_config.kv_transfer_config is not None, ( - "kv_transfer_config must be set for MoRIIOConnector" - ) - self.kv_transfer_config = vllm_config.kv_transfer_config - self.is_producer = self.kv_transfer_config.is_kv_producer - self.layer_to_spec = build_layer_to_spec(kv_cache_config) - # we transfer by layer, but store by kv cache group - self.layer_to_group = dict(kv_cache_config.transfer_group_index_by_layer) - self._transfer_layer_names = set(self.layer_to_group) - _, mamba_transfer_group_ids = _split_kv_cache_group_kinds(kv_cache_config) - self._num_mamba_transfer_groups = len(mamba_transfer_group_ids) - self._mamba_payload_index_by_layer = { - layer_name: group_id if self.mode == MoRIIOMode.WRITE else payload_index - for payload_index, group_id in enumerate(mamba_transfer_group_ids, start=1) - for layer_name in kv_cache_config.transfer_groups[group_id].layer_names - } - - if self.is_producer: - set_role(ROLE.PRODUCER) - else: - set_role(ROLE.CONSUMER) - # mori engine - self._rank = get_world_group().rank - self._local_rank = get_world_group().local_rank - self.tp_rank = self.moriio_config.tp_rank - self.dp_rank = self.moriio_config.dp_rank - - self.local_ip = self.moriio_config.local_ip - self.local_kv_port = self.moriio_config.local_kv_port - self.proxy_ip = self.moriio_config.proxy_ip - self.local_ping_port = self.moriio_config.local_ping_port - self.proxy_ping_port = self.moriio_config.proxy_ping_port - self.http_port = self.moriio_config.http_port - self.handshake_port = self.moriio_config.handshake_port - self.notify_port = self.moriio_config.notify_port - - self.zmq_context = zmq.Context() - self.metadata_address = ( - f"{self.moriio_config.local_ip}:{self.moriio_config.local_ping_port}" - ) - self.request_address = ( - f"{self.moriio_config.local_ip}:{self.moriio_config.http_port}" - ) - - self.moriio_engine = None - self._handle_request_thread = None - self._heartbeat: MoRIIOHeartbeat | None = None - self._writer = MoRIIOWriter(self) - # Completions that arrived before transfer_id_to_request_id was populated. - # Retried each step until the mapping is established. - self._unmatched_write_completions: set[str] = set() - self._unmatched_write_failures: OrderedDict[TransferId, None] = OrderedDict() - # Producer-side READ-mode ACK fan-in. When decode TP is larger than - # prefill TP, multiple decode ranks can read from one prefill rank and - # notify the same transfer_id. Blocks are reusable only after all ACKs. - self._consumer_notification_counts: dict[TransferId, int] = {} - self._completed_consumer_notifications: set[TransferId] = set() - - role = "producer" if self.is_producer else "consumer" - engine_suffix = ( - f"{self.moriio_config.local_ip}:{self.moriio_config.handshake_port}:" - f"tp{self.tp_rank}:dp{self.dp_rank}" - ) - self.moriio_engine = IOEngine( - f"{role}:{engine_suffix}", - IOEngineConfig( - self.moriio_config.local_ip, self.moriio_config.local_kv_port - ), - ) - logger.debug( - "build MORI IOEngine %s (ip=%s port=%s)", - f"{role}:{engine_suffix}", - self.moriio_config.local_ip, - self.moriio_config.local_kv_port, - ) - - if self._rank == 0 and self.moriio_config.proxy_ip: - self._heartbeat = self._start_heartbeat() - - logger.info( - "Initializing MoRIIO Engine, engine = %s, role = %s", - self.moriio_engine, - "producer" if self.is_producer else "consumer", - ) - - # Agent. - self.moriio_wrapper = MoRIIOWrapper( - tp_rank=self.tp_rank, - dp_rank=self.dp_rank, - transfer_timeout=self.moriio_config.transfer_timeout, - ) - self.moriio_wrapper.set_moriio_engine(self.moriio_engine) - backend = ( - BackendType.XGMI - if self.moriio_config.backend == "xgmi" - else BackendType.RDMA - ) - self.moriio_wrapper.set_backend_type( - backend, - qp_per_transfer=self.moriio_config.qp_per_transfer, - post_batch_size=self.moriio_config.post_batch_size, - num_workers=self.moriio_config.num_workers, - ) - self.moriio_wrapper.notify_port = self.moriio_config.notify_port - self.local_kv_cache_metadata: list[bytes] = [] - self.local_kv_cache_size: list[int] = [] - self.layer_name_to_local_kv_cache_metadata: dict[str, list[bytes]] = {} - - self.remote_kv_cache_metadata: list[bytes] = [] - self.remote_kv_cache_size: list[int] = [] - self.layer_name_to_remote_kv_cache_metadata: dict[str, dict[str, list[Any]]] = ( - dict() - ) - self.remote_moriio_metadata: dict[EngineId, MoRIIOAgentMetadata] = {} - self.slot_size_bytes = 0 - - self.load_ready_flag: dict[str, bool] = {} - self.write_ready_flags: dict[str, bool] = {} - self.kv_cache_shape = None - self.block_shape = None - self.kv_element_size = 0 - self.kv_cache_shapes: dict[str, torch.Size] = {} - self.block_lens: dict[str, int] = {} - - # Map of engine_id -> {agent_name0, agent_name1..}. - self._remote_agents: dict[EngineId, set[str]] = {} - - self.side_channel_port: int = ( - self.moriio_config.handshake_port - + get_port_offset(self.dp_rank, self.tp_rank) - ) - self.engine_id: EngineId = engine_id - - self.world_size = get_tensor_model_parallel_world_size() - self.tp_group = get_tp_group() - - # KV Caches and moriio tracking data. - self.kv_caches: dict[str, torch.Tensor] = {} - self.kv_layer_mr_offset: dict[str, int] = {} - self.layer_base_addr_index: dict[str, int] = {} - - # Map of engine_id -> kv_caches_base_addr. For TP case, each local - # rank will still only pull from a single remote TP worker. - self.kv_caches_base_addr: dict[EngineId, list[int]] = {} - - # Number of MoRIIO regions. Currently one region per cache - # (so 1 per layer for MLA, otherwise 2 per layer) - self.num_regions = 0 - self.num_layers = 0 - - # Map of engine_id -> num_blocks. All ranks in the same deployment will - # have the same number of blocks. - self.dst_num_blocks: dict[EngineId, int] = {} - # In-progress READ transfers: req_id -> {layer_name: [statuses]}. - # A KDA layer posts two reads (conv + ssm), so a layer maps to a list. - self._recving_transfers: defaultdict[ReqId, dict] = defaultdict(dict) - # Values are (remote_host, remote_notify_port, transfer_id). - self._recving_transfers_callback_addr: dict[ReqId, tuple[str, str, str]] = {} - # Monotonic-clock start times for each in-flight recv transfer. - # Used by _pop_done_transfers to abort transfers whose RDMA - # completion is lost, instead of hanging forever. - self._recving_transfers_start: dict[str, float] = {} - # Destination attention blocks per in-flight recv. Failed non-HMA reads - # report these to the scheduler's single-group recovery path. - self._recving_local_blocks: dict[ReqId, list[int]] = {} - # Drained by get_block_ids_with_load_errors. - self._invalid_block_ids: set[int] = set() - # This step's posted reads, awaited before the forward when the - # per-layer barrier cannot run. - self._reads_issued_this_step: list = [] - # This step's posted recurrent-state reads. Mamba layers are not - # wrapped by the attention KV-transfer decorator, so wait_for_layer_load - # never fires for them and these must be awaited in start_load_kv. - self._mamba_reads_this_step: list = [] - - # Track the expiration time of requests that are waiting to be sent. - self._reqs_to_send: dict[ReqId, float] = {} - - # Background thread for handling new handshake requests. - self._moriio_handshake_listener_t: threading.Thread | None = None - # Background thread for initializing new MoRIIO handshakes. - self._handshake_initiation_executor = ThreadPoolExecutor( - # MoRIIO is not guaranteed to be thread-safe, limit 1 worker. - max_workers=1, - thread_name_prefix="vllm-moriio-handshake-initiator", - ) - self._ready_requests = queue.Queue[tuple[ReqId, ReqMeta]]() - self._handshake_futures: dict[EngineId, Future[set[str]]] = {} - # Protects _handshake_futures and _remote_agents. - self._handshake_lock = threading.RLock() - # Remote engines already covered by the eager pre-forward handshake. - self._eager_handshaked_engines: set[EngineId] = set() - - self.block_size = vllm_config.cache_config.block_size - self.model_config = vllm_config.model_config - self.cache_config = vllm_config.cache_config - - self.block_window_per_layer: list[int | None] = [] - self.use_mla = self.model_config.use_mla - # Hybrid (mamba/KDA) recurrent-state transfer. Populated in - # register_kv_caches when the model has a MambaSpec kv_cache group. - self._has_mamba = False - self._conv_decomp: MambaConvSplitInfo | None = None - self._mamba_offset_templates: dict[str, MambaOffsetTemplate] = {} - # layer_name -> flat session indices (one per registered region: - # 1 for attention, 2 (conv, ssm) for a KDA layer). Built once lazily. - self._region_session_index: dict[str, list[int]] | None = None - self.built_session = False - self.built_write_session: defaultdict[str, list] = defaultdict(list) - backend = get_attn_backend( - self.model_config.get_head_size(), - self.model_config.dtype, - self.cache_config.cache_dtype, - use_mla=self.use_mla, - ) - self.transfer_id_to_request_id: dict[TransferId, ReqId] = {} - # READ-mode producer: a decode release-ACK can arrive BEFORE - # start_load_kv populates transfer_id_to_request_id (the notify races - # ahead of the scheduler->worker sync). Buffer such ACKs and retry them - # next get_finished tick instead of dropping them -- dropping loses the - # completion, so the request is never marked done_sending, its KV blocks - # leak, and the prefill KV cache wedges at high concurrency. Buffered - # BEFORE resolve_moriio_transfer_ack, so each ACK is counted exactly once - # (on the tick its mapping exists) -- the heterogeneous-TP ack-counting - # is preserved. - self._pending_unmapped_acks: list = [] - - # TODO: consider the integration of flashinfer or other backends. - self.backend_name = backend.get_name() - logger.debug("Detected attention backend %s", self.backend_name) - - def schedule_write_blocks( - self, - request_id: ReqId, - transfer_id: TransferId, - dst_engine_id: str, - local_block_ids: BlockIds, - remote_block_ids: BlockIds | None, - layer_name: str, - kv_layer: torch.Tensor, - remote_notify_port: int, - remote_ip: str, - multi_pod_hosts: list[str], - remote_dp_size_local: int, - remote_dp_size: int, - remote_tp_size: int | None = None, - ) -> None: - """Schedule a block write operation. - - Args: - request_id: Unique identifier for the request - transfer_id: Unique identifier for the transfer - dst_engine_id: Destination engine ID - local_block_ids: Local block IDs to transfer - remote_block_ids: Hint for remote block IDs - layer_name: Name of the layer - kv_layer: KV cache tensor - remote_notify_port: Port for completion notification - remote_ip: IP address of remote node - multi_pod_hosts: List of pod IPs for multi-pod Wide-EP - remote_dp_size_local: Per-pod DP size for multi-pod - remote_dp_size: Global DP size of the decode instance - remote_tp_size: Tensor parallel size advertised by the peer - - """ - # synchronization to prevent dirty reads between - # transfer and attention operations - # we can consider removing this synchronization after ibgda is enabled. - # when mori-io supports ibgda functionality - - stream = torch.cuda.current_stream() - event = torch.cuda.Event() - event.record(stream) - - task = WriteTask( - request_id=request_id, - transfer_id=transfer_id, - dst_engine_id=dst_engine_id, - local_block_ids=local_block_ids, - remote_block_ids_hint=remote_block_ids, - layer_name=layer_name, - event=event, - remote_notify_port=remote_notify_port, - remote_ip=remote_ip, - multi_pod_hosts=multi_pod_hosts, - remote_dp_size_local=remote_dp_size_local, - remote_dp_size=remote_dp_size, - remote_tp_size=remote_tp_size, - ) - self._writer.schedule_write(task) - - def _get_built_session(self, remote_engine_id): - if remote_engine_id not in self.built_write_session: - remote_layer_map = self.layer_name_to_remote_kv_cache_metadata[ - remote_engine_id - ] - cur_remote_engine_sessions = [] - for ln, local_metas in self.layer_name_to_local_kv_cache_metadata.items(): - remote_metas = remote_layer_map[ln] - if len(remote_metas) != len(local_metas): - # Sessions are addressed by region index, so a peer that - # registered a different number of regions for this layer - # would not just lose a region to zip(): every later index - # would point into the wrong layer's session. - raise MoRIIOError( - f"peer {remote_engine_id} registered " - f"{len(remote_metas)} region(s) for layer {ln}, this " - f"engine has {len(local_metas)}; the two sides must run " - "the same model and KV cache layout" - ) - # One session per registered region, in registration order: - # attention layers have a single region; KDA layers have two - # (conv then ssm). This flat layout is indexed via - # _region_session_indices(layer_name). - for local_meta, remote_meta in zip(local_metas, remote_metas): - unpacked_local_memory_meta = ( - self.moriio_wrapper.get_unpack_memory_metadata(local_meta) - ) - unpacked_remote_memory_meta = ( - self.moriio_wrapper.get_unpack_memory_metadata(remote_meta) - ) - cur_remote_engine_sessions.append( - self.moriio_wrapper.build_session( - unpacked_local_memory_meta, unpacked_remote_memory_meta - ) - ) - self.built_write_session[remote_engine_id] = cur_remote_engine_sessions - return self.built_write_session[remote_engine_id], self.remote_moriio_metadata[ - remote_engine_id - ] - - def _start_heartbeat(self) -> MoRIIOHeartbeat: - payload = { - "type": "P" if self.is_producer else "D", - "http_address": self.request_address, - "zmq_address": ( - f"host:{self.local_ip},handshake:{self.handshake_port}," - f"notify:{self.notify_port}" - ), - "dp_size": self.moriio_config.dp_size, - "tp_size": self.moriio_config.tp_size, - "transfer_mode": self.mode.name, - } - return MoRIIOHeartbeat( - f"tcp://{self.proxy_ip}:{self.proxy_ping_port}", - payload, - MoRIIOConstants.PING_INTERVAL, - MoRIIOConstants.MAX_PING_RETRIES, - ) - - def shutdown(self): - heartbeat = getattr(self, "_heartbeat", None) - if heartbeat is not None: - heartbeat.shutdown() - self._heartbeat = None - - if hasattr(self, "moriio_wrapper") and self.moriio_wrapper: - self.moriio_wrapper.shutdown() - - if hasattr(self, "_handshake_initiation_executor"): - self._handshake_initiation_executor.shutdown(wait=False) - - if ( - hasattr(self, "_moriio_handshake_listener_t") - and self._moriio_handshake_listener_t - ): - self._moriio_handshake_listener_t.join(timeout=0) - - if hasattr(self, "zmq_context") and self.zmq_context: - self.zmq_context.destroy(linger=0) - self.zmq_context = None - - def __del__(self): - self.shutdown() - - @staticmethod - def _moriio_handshake_listener( - metadata: MoRIIOAgentMetadata, - ready_event: threading.Event, - base_port: int, - tp_rank: int, - dp_rank: int, - layer_name_to_local_kv_cache_metadata: dict, - ): - """Background thread for getting new MoRIIO handshakes.""" - encoder = msgspec.msgpack.Encoder() - encoded_data = encoder.encode(metadata) - size_in_bytes = len(encoded_data) - logger.debug( - "Size of encoded MoRIIOAgentMetadata: %s bytes", str(size_in_bytes) - ) - - # Listen for new requests for metadata. - host = "*" - - path = make_zmq_path("tcp", host, base_port) - logger.debug("mori handshake starting listening on path: %s", path) - - with zmq_ctx(zmq.ROUTER, path) as sock: - ready_event.set() - while True: - identity, msg = sock.recv_multipart() - if ( - msg != MoRIIOConstants.GET_META_MSG - and msg != MoRIIOConstants.POP_DONE_RECV - ): - logger.error("Connection listener got unexpected message") - raise HandshakeError("handshake failed, unexpected msg type") - elif msg == MoRIIOConstants.GET_META_MSG: - sock.send_multipart( - (identity, b"", encoded_data) - ) # send local mori io engine meta data - logger.debug("MoRIIO handshake listener sent metadata") - # now we send tensor meta data for each block - buf = msgpack.dumps(layer_name_to_local_kv_cache_metadata) - sock.send_multipart((identity, b"", buf)) - elif msg == MoRIIOConstants.POP_DONE_RECV: - _, req_id = sock.recv_multipart() - logger.debug( - "MoRIIO handshake listener received done recv for req", - req_id.decode(), - ) - - def _moriio_handshake( - self, - host: str, - port: int, - remote_tp_size: int, - expected_engine_id: str, - remote_dp_rank: int = 0, - remote_tp_rank: int | None = None, - ) -> set[str]: - """Do a MoRIIO handshake with a remote instance. - - remote_tp_rank: explicit remote TP index to dial. Flexible-read callers - pass the chosen prefill TP rank so the handshake, the (dp, tp) session - key and the notify port all address the SAME rank. None falls back to the - local-rank mapping _remote_tp_rank -- byte-identical for callers not yet - TP-aware. - """ - start_time = time.perf_counter() - - # NOTE(rob): we need each rank to have a unique port. This is - # a hack to keep us moving. We will switch when moving to etcd - # or where we have a single ZMQ socket in the scheduler. - - dial_tp_rank = ( - self._remote_tp_rank(remote_tp_size) - if remote_tp_rank is None - else int(remote_tp_rank) - ) - port_offset = get_port_offset(remote_dp_rank, dial_tp_rank, remote_tp_size) - path = make_zmq_path("tcp", host, port + port_offset) - logger.debug("handshake Querying metadata on path: %s", path) - - # Send query for the request. - with zmq_ctx(zmq.DEALER, path) as sock: - logger.debug("prepare send msg INSTAZNCE: %s", path) - sock.send(MoRIIOConstants.GET_META_MSG) - received_frame = sock.recv_multipart() - if len(received_frame) != 2 or received_frame[0] != b"": - raise HandshakeError(f"Unexpected frame! {received_frame = }") - - metadata_bytes = received_frame[1] - decoder = msgspec.msgpack.Decoder(MoRIIOAgentMetadata) - metadata = decoder.decode(metadata_bytes) - got_metadata_time = time.perf_counter() - logger.info( - "MoRIIO handshake: get metadata took: %s", - got_metadata_time - start_time, - ) - - self.moriio_wrapper.remote_engine_ip = host - remote_agent_name = self.moriio_wrapper.register_remote_engine( - metadata.agent_metadata - ) - - logger.debug( - "MoRIIO handshake: registered" - "remote agent %s for engine ID %s, path = %s", - remote_agent_name, - expected_engine_id, - path, - ) - - if len(self.local_kv_cache_metadata) > 0: - logger.warning( - "len(self.local_kv_cache_metadata) = %s," - "maybe you didnt clear this buffer correctly", - len(self.local_kv_cache_metadata), - ) - self.local_kv_cache_metadata = [] - if len(self.remote_kv_cache_metadata) > 0: - logger.warning( - "len(self.remote_kv_cache_metadata) = %s," - "maybe you didnt clear this buffer correctly", - len(self.remote_kv_cache_metadata), - ) - self.remote_kv_cache_metadata = [] - - received_frame = sock.recv_multipart() - if len(received_frame) != 2 or received_frame[0] != b"": - raise HandshakeError(f"unexpected frame! {received_frame = }") - buf = received_frame[1] - self.layer_name_to_remote_kv_cache_metadata[expected_engine_id] = ( - msgpack.loads(buf) - ) - self.remote_moriio_metadata[expected_engine_id] = metadata - setup_agent_time = time.perf_counter() - logger.debug( - "MoRIIO handshake: add agent took: %s", - setup_agent_time - got_metadata_time, - ) - - return {remote_agent_name} - - def _remote_tp_rank(self, remote_tp_size: int) -> int: - # 0/unknown remote TP == homogeneous (avoids collapsing all ranks to 0). - if remote_tp_size == 0: - remote_tp_size = self.world_size - return get_moriio_remote_tp_rank(self.tp_rank, self.world_size, remote_tp_size) - - def _background_moriio_handshake( - self, req_id: ReqId, remote_engine_id: EngineId, meta: ReqMeta - ): - # Do MoRIIO handshake in background and add to _ready_requests when done. - fut = None - if remote_engine_id is not None: - fut = self._handshake_futures.get(remote_engine_id) - if fut is None: - host = meta.remote_host - port = int(meta.remote_handshake_port) - tp_size = int(meta.tp_size) - remote_dp_size = int(meta.remote_dp_size) - # Wide-EP multi-pod: remote DP ranks span pods at different IPs - # (ranks per pod = dp_local), so resolve the host per cur_dp_rank - # below instead of using a single host for all ranks. - pod_hosts = list(meta.multi_pod_hosts) if meta.multi_pod_hosts else [host] - remote_dp_size_local = int(meta.remote_dp_size_local) or remote_dp_size - - def request_ready(_f: Future[Any], entry=(req_id, meta)): - logger.info("MoRIIO handshake done for request %s", req_id) - self._ready_requests.put(entry) - self.load_ready_flag[remote_engine_id] = True - self.write_ready_flags[remote_engine_id] = True - - fut_list = [] - - # In dp(prefill)<->dp(decode) communication, we require an all-to-all handshake. - - for cur_dp_rank in range(remote_dp_size): - dp_engine_id = self.get_engine_name_with_dp(remote_engine_id, cur_dp_rank) - _pod_idx = pod_index(cur_dp_rank, remote_dp_size_local) - if _pod_idx >= len(pod_hosts): - _pod_idx = 0 - _per_rank_host = pod_hosts[_pod_idx] - # The handshake port offset must use the per-pod local rank, since - # each pod binds sockets only for its local ranks; dp_engine_id - # keeps the global rank for uniqueness. Single-pod is bit-identical. - _per_rank_local_dp = fold_local_rank(cur_dp_rank, remote_dp_size_local) - future = self._handshake_initiation_executor.submit( - self._moriio_handshake, - _per_rank_host, - port, - tp_size, - dp_engine_id, - _per_rank_local_dp, - ) - fut_list.append(future) - - def done_callback(f: Future[set[str]], eid=dp_engine_id): - with self._handshake_lock: - self._handshake_futures.pop(eid, None) - try: - self._remote_agents[eid] = f.result() - except Exception: - logger.exception("Handshake with %s failed", eid) - - future.add_done_callback(done_callback) - self._handshake_futures[dp_engine_id] = future - - # fut = fut_list - def wait_all_dp(): - for future in fut_list: - future.result() - return True - - all_done_future = self._handshake_initiation_executor.submit(wait_all_dp) - all_done_future.add_done_callback(request_ready) - - def _is_mla_cache_layer(self, layer_name: str) -> bool: - return is_mla_cache_layer(self.layer_to_spec, layer_name) - - def _is_mamba_layer(self, layer_name: str) -> bool: - """True for a hybrid/KDA layer backed by a packed state page.""" - return isinstance(self.layer_to_spec.get(layer_name), MambaSpec) - - @staticmethod - def _contiguous_byte_alias(tensor: torch.Tensor) -> torch.Tensor: - """Return a contiguous uint8 view over ``tensor``'s full byte extent. - - KDA conv/ssm caches are slot-strided views (``stride(0)`` may exceed the - per-slot element count), so they are non-contiguous and cannot be passed - to ``register_torch_tensor``. This aliases the same storage (zero-copy, - identical ``data_ptr``) as a flat uint8 tensor spanning - ``shape[0] * stride(0)`` elements -- the full slot-strided extent that - transfer offsets (``slot * stride(0) * elem``) address. - """ - esz = tensor.element_size() - storage_nbytes = tensor.untyped_storage().nbytes() - offset_bytes = tensor.storage_offset() * esz - extent_bytes = tensor.shape[0] * tensor.stride(0) * esz - # Clamp to the bytes remaining from the view's storage offset. In a - # packed page, the ssm view starts after conv, so its slot-strided extent - # would otherwise run past the backing allocation. - extent_bytes = min(extent_bytes, storage_nbytes - offset_bytes) - assert extent_bytes > 0, ( - "KDA byte alias empty: storage_offset=" - f"{tensor.storage_offset()} * esz={esz} >= " - f"storage_nbytes={storage_nbytes}" - ) - alias = torch.empty(0, dtype=torch.uint8, device=tensor.device) - alias.set_( - tensor.untyped_storage(), - tensor.storage_offset() * esz, - (extent_bytes,), - (1,), - ) - return alias - - def _get_layer_transfer_geometry( - self, layer_name: str, remote_num_blocks: int | None = None - ) -> LayerTransferGeometry: - geometry = get_layer_transfer_geometry( - layer_name, - self.kv_caches[layer_name], - self.layer_to_spec, - remote_num_blocks, - ) - # get_layer_transfer_geometry also serves mamba layers, whose geometry - # is a different NamedTuple with no block fields. Every caller here is - # on the attention path (mamba layers return before reaching it), so - # narrow the type instead of leaking the union. - assert isinstance(geometry, LayerTransferGeometry), ( - f"layer {layer_name} is a mamba layer; use get_mamba_transfer_geometry" - ) - return geometry - - def _iter_layer_registration_regions( - self, layer_name: str - ) -> list[tuple[torch.Tensor, int]]: - return iter_layer_registration_regions( - layer_name, - self.kv_caches[layer_name], - self.layer_to_spec, - ) - - def _build_shared_kv_mr( - self, kv_caches: dict[str, torch.Tensor] - ) -> tuple[torch.Tensor, dict[str, int]]: - page_size = 4096 - backing = next(iter(kv_caches.values())).view(torch.uint8) - nbytes = backing.untyped_storage().nbytes() - base = torch.as_strided(backing, (nbytes,), (1,), 0) - ptr = base.data_ptr() - slack = ptr % page_size - end = 0 - for layer_name in kv_caches: - _, region_len = next( - iter(self._iter_layer_registration_regions(layer_name)) - ) - end = max(end, kv_caches[layer_name].data_ptr() - ptr + region_len) - reg_nbytes = ((slack + end + page_size - 1) // page_size) * page_size - if reg_nbytes > nbytes: - raise ValueError( - f"shared KV backing too small for page-aligned MR: " - f"need {reg_nbytes}, have {nbytes}" - ) - reg = torch.as_strided(base, (reg_nbytes,), (1,), 0 if slack == 0 else -slack) - mr_ptr = reg.data_ptr() - return reg, { - name: cache.data_ptr() - mr_ptr for name, cache in kv_caches.items() - } - - def _remote_layer_mr_offset( - self, - layer_name: str, - remote_moriio_meta: MoRIIOAgentMetadata, - remote_engine_id: EngineId | None, - ) -> int: - if not self.kv_layer_mr_offset or not remote_engine_id: - return 0 - idx = self.layer_base_addr_index[layer_name] - metas = self.layer_name_to_remote_kv_cache_metadata.get( - remote_engine_id, {} - ).get(layer_name) - if not metas or idx >= len(remote_moriio_meta.kv_caches_base_addr): - return 0 - mr_base = self.moriio_wrapper.get_unpack_memory_metadata(metas[0]).data - return remote_moriio_meta.kv_caches_base_addr[idx] - mr_base - - def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]): - """Register the KV Cache data in moriio.""" - kv_caches = { - layer_name: kv_cache - for layer_name, kv_cache in kv_caches.items() - if layer_name in self._transfer_layer_names - } - if not kv_caches: - raise MoRIIOError("MoRIIO has no transfer-enabled KV cache layers") - self.kv_caches = kv_caches # layer name to kv cache - self._has_mamba = any( - self._is_mamba_layer(layer_name) for layer_name in kv_caches - ) - if self._has_mamba: - mamba_specs = [ - cast(MambaSpec, self.layer_to_spec[layer_name]) - for layer_name in kv_caches - if self._is_mamba_layer(layer_name) - ] - mamba_spec = _validate_mamba_specs(mamba_specs) - assert mamba_spec is not None - - from vllm.model_executor.layers.mamba.mamba_utils import ( - is_conv_state_dim_first, - ) - - if not is_conv_state_dim_first(): - raise MoRIIOError( - "MoRIIO KDA conv transfer requires the DS (dim-first) " - "state layout; set VLLM_SSM_CONV_STATE_LAYOUT=DS" - ) - self._conv_decomp = derive_mamba_conv_split(mamba_spec, self.world_size) - # KDA layers use packed pages with different geometry; only record - # attention tensor shapes here. - self.kv_cache_shapes = { - layer_name: kv_cache.shape - for layer_name, kv_cache in kv_caches.items() - if not self._is_mamba_layer(layer_name) - } - - # Geometry selection must pick an attention layer (never a KDA tuple): - # prefer a standard 5-D K/V layer, else the first attention layer. - attn_items = [ - (layer_name, kv_cache) - for layer_name, kv_cache in kv_caches.items() - if not self._is_mamba_layer(layer_name) - ] - first_layer_name, first_kv_cache = next( - ( - (layer_name, kv_cache) - for layer_name, kv_cache in attn_items - if ( - not self._is_mla_cache_layer(layer_name) - and len(kv_cache.shape) == 5 - and (kv_cache.shape[0] == 2 or kv_cache.shape[1] == 2) - ) - ), - attn_items[0] if attn_items else next(iter(kv_caches.items())), - ) - kv_elem_size = first_kv_cache.element_size() - - use_mla = self._is_mla_cache_layer(first_layer_name) - first_geometry = self._get_layer_transfer_geometry(first_layer_name) - - if use_mla: - # MLA case. - block_rank = 2 # [block_size, latent_dim] - block_shape = first_kv_cache.shape[-block_rank:] - else: - # [2, num_blocks, ...] or [num_blocks, 2, ...] - block_rank = 3 # [block_size, kv_heads, head_dim] - block_shape = first_kv_cache.shape[-block_rank:] - self.num_blocks = first_geometry.num_blocks - self.slot_size_bytes = first_geometry.slot_size_bytes - if first_geometry.block_size != self.block_size: - # DeepSeek-V3 / MLA backends (e.g. FlashMLA) override the - # configured block_size at attention-layer creation time, so - # the KV cache tensor is laid out with a different (usually - # larger) block_size than vllm_config.cache_config.block_size. - # Trust the actual tensor shape and reconcile. - logger.info( - "KV cache block_size=%d differs from config block_size=%d; " - "using actual tensor shape (attention backend override).", - first_geometry.block_size, - self.block_size, - ) - self.block_size = first_geometry.block_size - # block size in bytes - self.block_len = first_geometry.block_len - self.kv_cache_shape = first_kv_cache.shape - self.block_shape = block_shape - self.kv_element_size = kv_elem_size - - self.dst_num_blocks[self.engine_id] = self.num_blocks - kv_caches_base_addr = [] - caches_data = [] - self.layer_base_addr_index = {} - base_addr_idx = 0 - # HMA (#53721 Gemma4 SWA, GLM-5.2 DSA, MiniMax): transfers already - # size RDMA from per-layer geometry, so allow divergent block_len - # (bytes) always, and divergent block_size (tokens/page) only when - # the hybrid KV cache manager is enabled. Without HMA, block_size - # must stay uniform for the peer page handshake. - hma_enabled = ( - not self.vllm_config.scheduler_config.disable_hybrid_kv_cache_manager - ) - - for layer_name in kv_caches: - self.layer_base_addr_index[layer_name] = base_addr_idx - if self._is_mamba_layer(layer_name): - # KDA layer: two whole-tensor regions (conv, ssm); no block-size - # geometry (recurrent state is per-slot, not paged). - for cache, region_len in self._iter_layer_registration_regions( - layer_name - ): - base_addr = cache.data_ptr() - caches_data.append((base_addr, region_len, cache.device.index, "")) - kv_caches_base_addr.append(base_addr) - base_addr_idx += 1 - continue - geometry = self._get_layer_transfer_geometry(layer_name) - if geometry.block_size != self.block_size: - if not hma_enabled: - raise ValueError( - "MoRIIO KV cache block size mismatch for layer " - f"{layer_name}: {geometry.block_size} != " - f"{self.block_size}" - ) - logger.debug( - "MoRIIO HMA per-layer block_size for %s: %d " - "(page/default %d); continuing with per-layer sizes.", - layer_name, - geometry.block_size, - self.block_size, - ) - if geometry.block_len != self.block_len: - # Unlike block_size above, divergent block_len is allowed - # unconditionally (not gated on hma_enabled) -- it covers - # non-hybrid models too, e.g. GLM-5.2 DSA's per-layer - # geometry. Don't attribute it to HMA in the log. - logger.debug( - "MoRIIO per-layer block_len for %s: %d " - "(page/default %d); continuing with per-layer sizes.", - layer_name, - geometry.block_len, - self.block_len, - ) - # num_blocks is advertised as a single scalar to the peer, so must it - # be uniform - if geometry.num_blocks != self.num_blocks: - raise ValueError( - "MoRIIO KV cache num_blocks mismatch for layer " - f"{layer_name}: {geometry.num_blocks} != {self.num_blocks}" - ) - self.block_lens[layer_name] = geometry.block_len - for cache, region_len in self._iter_layer_registration_regions(layer_name): - base_addr = cache.data_ptr() - caches_data.append((base_addr, region_len, cache.device.index, "")) - kv_caches_base_addr.append(base_addr) - base_addr_idx += 1 - - shared_backing = ( - all(isinstance(t, torch.Tensor) for t in kv_caches.values()) - and len({id(t.untyped_storage()) for t in kv_caches.values()}) == 1 - ) - shared_mr = None - if shared_backing: - reg_tensor, self.kv_layer_mr_offset = self._build_shared_kv_mr(kv_caches) - shared_mr = self.moriio_wrapper.register_local_tensor(reg_tensor) - - # Say it out loud when the groups disagree. This is expected on a hybrid - # model and used to be fatal, so a run that silently had one page size - # and a run that has three should not look identical in the log. - distinct_block_lens = sorted(set(self.block_lens.values())) - if len(distinct_block_lens) > 1: - self.block_len = max(distinct_block_lens) - logger.info( - "MoRIIO registered %d attention layers with %d distinct block " - "lengths (bytes): %s; transfer sizes and strides are per layer, " - "and %d is the page advertised to the peer.", - len(self.block_lens), - len(distinct_block_lens), - distinct_block_lens, - self.block_len, - ) - - for layer_name, kv_cache in kv_caches.items(): - if layer_name not in self.layer_name_to_local_kv_cache_metadata: - self.layer_name_to_local_kv_cache_metadata[layer_name] = [] - - if self._is_mamba_layer(layer_name): - # Register conv and ssm as two whole-tensor regions, appending - # both metadata blobs (order: conv then ssm) so the session - # layout matches iter_layer_registration_regions. conv/ssm are - # slot-strided views and are not contiguous, which - # register_torch_tensor rejects; register a contiguous uint8 - # alias over each tensor's full byte extent instead (zero-copy: - # same storage and data_ptr). - conv, ssm = kda_conv_ssm( - kv_cache, cast(MambaSpec, self.layer_to_spec[layer_name]) - ) - logger.debug( - "MoRIIO KDA register %s: conv shape=%s stride=%s contig=%s " - "storage_nbytes=%s ; ssm shape=%s stride=%s contig=%s " - "storage_nbytes=%s", - layer_name, - tuple(conv.shape), - tuple(conv.stride()), - conv.is_contiguous(), - conv.untyped_storage().nbytes(), - tuple(ssm.shape), - tuple(ssm.stride()), - ssm.is_contiguous(), - ssm.untyped_storage().nbytes(), - ) - for tensor in (conv, ssm): - alias = self._contiguous_byte_alias(tensor) - meta = self.moriio_wrapper.register_local_tensor(alias) - self.layer_name_to_local_kv_cache_metadata[layer_name].append(meta) - self.local_kv_cache_size.append(alias.numel()) - else: - moriio_mem_metadata = shared_mr or ( - self.moriio_wrapper.register_local_tensor(kv_cache) - ) - self.layer_name_to_local_kv_cache_metadata[layer_name].append( - moriio_mem_metadata - ) - self.local_kv_cache_size.append( - kv_cache.nelement() * kv_cache.element_size() - ) - - self.kv_caches_base_addr[self.engine_id] = kv_caches_base_addr - self.num_regions = len(caches_data) - self.num_layers = len(self.kv_caches.keys()) - - # Report the prevalidated KDA decomposition after registration. - if self._has_mamba: - assert self._conv_decomp is not None - logger.info( - "MoRIIO registered %d KDA (conv+ssm) layer(s); ssm sizes=%s", - sum(1 for ln in kv_caches if self._is_mamba_layer(ln)), - self._conv_decomp.ssm_sizes, - ) - # Optimization for models with local attention (Llama 4) - if self.vllm_config.model_config.hf_config.model_type == "llama4": - from transformers import Llama4TextConfig - - assert isinstance( - self.vllm_config.model_config.hf_text_config, Llama4TextConfig - ) - llama4_config = self.vllm_config.model_config.hf_text_config - no_rope_layers = llama4_config.no_rope_layers - chunk_size = llama4_config.attention_chunk_size - chunk_block_size = math.ceil(chunk_size / self.block_size) - for layer_idx in range(self.num_layers): - # no_rope_layers[layer_idx] == 0 means NoPE (global) - # Any other value means RoPE (local chunked) - is_local_attention = no_rope_layers[layer_idx] != 0 - block_window = chunk_block_size if is_local_attention else None - self.block_window_per_layer.append(block_window) - logger.debug( - "Llama 4 block window per layer mapping: %s", - self.block_window_per_layer, - ) - assert len(self.block_window_per_layer) == self.num_layers - - metadata = MoRIIOAgentMetadata( - engine_id=self.engine_id, - agent_metadata=self.moriio_wrapper.get_agent_metadata(), - kv_caches_base_addr=self.kv_caches_base_addr[self.engine_id], - num_blocks=self.num_blocks, - block_len=self.block_len, - attn_backend_name=self.backend_name, - ) - ready_event = threading.Event() - self._moriio_handshake_listener_t = threading.Thread( - target=self._moriio_handshake_listener, - args=( - metadata, - ready_event, - self.side_channel_port, - self.tp_rank, - self.dp_rank, - self.layer_name_to_local_kv_cache_metadata, - ), - daemon=True, - name="moriio_handshake_listener", - ) - self._moriio_handshake_listener_t.start() - ready_event.wait() # Wait for listener ZMQ socket to be ready. - self.moriio_wrapper.async_wait_reqid() - - def get_finished(self) -> tuple[set[str], set[str]]: - """Get requests that are done sending or recving on this specific worker. - The scheduler process (via the MultiprocExecutor) will use this output - to track which workers are done. - """ - heartbeat = getattr(self, "_heartbeat", None) - if heartbeat is not None: - heartbeat.check_health() - done_sending, done_recving = set(), set() - - if self.is_producer: - # pop_finished_req_ids returns release ACKs sent by decode. Keep - # duplicate ACKs because heterogeneous TP can fan multiple decode - # ranks into one prefill rank for the same transfer_id. - # Combine freshly-arrived ACKs with any buffered from prior ticks - # whose transfer_id wasn't mapped yet (notify raced ahead of - # start_load_kv); retry the lookup every tick. Buffered before - # resolve_moriio_transfer_ack so each ACK is counted exactly once. - finished_acks = self._pending_unmapped_acks + list( - self.moriio_wrapper.pop_finished_req_ids() - ) - self._pending_unmapped_acks = [] - resolved_transfer_ids: set[TransferId] = set() - for ack in finished_acks: - transfer_id = ack if isinstance(ack, str) else ack.transfer_id - if transfer_id not in self.transfer_id_to_request_id: - # Mapping not populated yet -- buffer and retry next tick, - # do NOT drop (dropping leaks producer KV at high conc and - # wedges the prefill). - self._pending_unmapped_acks.append(ack) - continue - resolved_transfer_id = resolve_moriio_transfer_ack( - ack, - producer_tp_size=self.world_size, - live_transfer_ids=self.transfer_id_to_request_id.keys(), - notification_counts=self._consumer_notification_counts, - completed_transfer_ids=(self._completed_consumer_notifications), - ) - if resolved_transfer_id is not None: - resolved_transfer_ids.add(resolved_transfer_id) - done_sending = { - self.transfer_id_to_request_id[xfer_id] - for xfer_id in resolved_transfer_ids - } - else: - if self.mode == MoRIIOMode.WRITE: - fresh = self.moriio_wrapper.pop_finished_write_req_ids() - # Accumulate with any completions that arrived before their - # transfer_id was registered in transfer_id_to_request_id. - self._unmatched_write_completions |= fresh - done_recving = self._unmatched_write_completions - else: - # READ mode: the scheduler treats KV loads as synchronous - # (load_kv_async=False), so requests go directly to RUNNING - # instead of WAITING_FOR_REMOTE_KVS. We still call - # _pop_done_transfers() to send the notify to the prefill - # side and clean up internal state, but we must NOT report - # these as done_recving because the scheduler doesn't - # expect a finished_recving signal for RUNNING requests. - self._pop_done_transfers() - - done_recving = { - self.transfer_id_to_request_id[id] - for id in filter( - lambda id: id in self.transfer_id_to_request_id, done_recving - ) - } - if self.mode == MoRIIOMode.WRITE and not self.is_producer: - # Remove the ones we successfully matched; leave unmatched for retry. - matched_xfer_ids = { - id - for id in self._unmatched_write_completions - if id in self.transfer_id_to_request_id - } - self._unmatched_write_completions -= matched_xfer_ids - - return done_sending, done_recving - - def get_transfer_results(self) -> KVConnectorTransferResults: - done_sending, done_recving = self.get_finished() - failed_recving: set[ReqId] = set() - - if self.mode == MoRIIOMode.WRITE and not self.is_producer: - pending = self._unmatched_write_failures - mapping = self.transfer_id_to_request_id - for transfer_id in self.moriio_wrapper.pop_failed_write_req_ids(): - pending[transfer_id] = None - for transfer_id in [t for t in pending if t in mapping]: - del pending[transfer_id] - failed_recving.add(mapping[transfer_id]) - while len(pending) > _MAX_UNMATCHED_WRITE_FAILURES: - pending.popitem(last=False) - done_recving |= failed_recving - - return KVConnectorTransferResults( - finished_sending=done_sending, - finished_recving=done_recving, - failed_recving=failed_recving, - ) - - def wait_for_layer_load(self, layer_name: str) -> None: - """Block until all in-flight READs of this layer have landed. - - A host-side blocking wait must not run during full-graph capture. - """ - if self.is_producer or self.mode != MoRIIOMode.READ: - return - - if get_forward_context().cudagraph_runtime_mode == CUDAGraphMode.FULL: - return - - with self.moriio_wrapper.lock: - pending = [ - status - for status_by_layer in self._recving_transfers.values() - for status in status_by_layer.get(layer_name, ()) - ] - - if not pending: - return - - try: - self.moriio_wrapper.waiting_for_transfer_complete(pending) - except TransferError: - if self._has_mamba: - logger.exception( - "MoRIIO hybrid READ failed before layer %s; aborting the " - "forward because HMA load-error recovery is unsupported.", - layer_name, - ) - raise - logger.warning( - "MoRIIO READ barrier did not complete for layer %s; proceeding " - "to transfer cleanup.", - layer_name, - exc_info=True, - ) - - def _pop_done_transfers(self) -> set[str]: - done_req_ids: set[str] = set() - _xfer_timeout = self.moriio_config.recv_abort_timeout - with self.moriio_wrapper.lock: - to_remove = [] - for req_id, status_by_layer in self._recving_transfers.items(): - # Every layer's read must land before the request is complete, - # so the whole batch is polled rather than just the newest - # status. - statuses = [ - status - for layer_statuses in status_by_layer.values() - for status in layer_statuses - ] - state = self.moriio_wrapper.poll_transfer_batch(statuses) - if statuses and state is TransferBatchState.DONE: - host, port, xfer_id = self._recving_transfers_callback_addr[req_id] - done_req_ids.add(xfer_id) - self.moriio_wrapper.send_notify( - xfer_id, - host, - port, - message_type="release", - message_fields={"consumer_tp_size": self.world_size}, - ) - to_remove.append(req_id) - elif state is TransferBatchState.FAILED: - failed_status = next( - (status for status in statuses if status.Failed()), None - ) - logger.error( - "RDMA transfer failed for request %s: %s (code=%s). " - "Notifying prefill to free blocks and recording the " - "load error where scheduler recovery is supported.", - req_id, - failed_status.Message() if failed_status else "unknown", - failed_status.Code() if failed_status else "unknown", - ) - self._record_failed_recv(req_id) - host, port, xfer_id = self._recving_transfers_callback_addr[req_id] - try: - self.moriio_wrapper.send_notify( - xfer_id, - host, - port, - message_type="release", - message_fields={"consumer_tp_size": self.world_size}, - ) - except Exception: - logger.exception( - "Failed to send error notification for request %s", - req_id, - ) - to_remove.append(req_id) - # Deliberately not in done_req_ids: the decode KV is - # incomplete, so this is a load error rather than a - # completion. - elif req_id in self._recving_transfers_start: - # Abort still-in-flight transfers that exceed the - # configured deadline. Otherwise a lost RDMA - # completion would leave the decode worker hung - # indefinitely on this request. - _age = time.monotonic() - self._recving_transfers_start[req_id] - if _age > _xfer_timeout: - logger.error( - "RDMA read TIMED OUT for req %s after %.1fs " - "(kv_connector_extra_config.recv_abort_timeout=%.0f)", - req_id, - _age, - _xfer_timeout, - ) - self._record_failed_recv(req_id) - to_remove.append(req_id) - for req_id in to_remove: - del self._recving_transfers[req_id] - del self._recving_transfers_callback_addr[req_id] - self._recving_transfers_start.pop(req_id, None) - self._recving_local_blocks.pop(req_id, None) - - return done_req_ids - - def _await_reads_issued_this_step(self) -> None: - """Step-level READ barrier, for the reads no per-layer barrier covers. - - load_kv_async=False promises the KV is in place before the forward runs. - wait_for_layer_load keeps that promise per layer, letting early layers - overlap later transfers, but it only fires for layers that go through - the attention KV-transfer decorator. Two cases fall through it: - - * mamba/KDA layers have no such hook at all, so their conv+ssm reads - would otherwise be consumed while still in flight; - * under CUDAGraphMode.FULL a host-side blocking wait is illegal - anywhere in the graph, so every read is awaited here instead. - - start_load_kv runs outside the graph, so blocking here is safe. - Attention reads keep their per-layer overlap in the non-FULL case. - - Attention-only failures are reported via - get_block_ids_with_load_errors. Hybrid failures are re-raised because - the scheduler's recovery path does not support multiple cache groups; - this fails the step closed instead of running forward with incomplete - recurrent state. - """ - all_statuses = self._reads_issued_this_step - self._reads_issued_this_step = [] - mamba_statuses = self._mamba_reads_this_step - self._mamba_reads_this_step = [] - if get_forward_context().cudagraph_runtime_mode == CUDAGraphMode.FULL: - statuses = all_statuses - else: - statuses = mamba_statuses - if not statuses: - return - try: - self.moriio_wrapper.waiting_for_transfer_complete(statuses) - except TransferError: - if self._has_mamba: - logger.exception( - "MoRIIO hybrid READ failed before the forward; aborting " - "because HMA load-error recovery is unsupported." - ) - raise - logger.exception( - "MoRIIO reads did not complete before the forward; proceeding " - "to transfer cleanup." - ) - - def _record_failed_recv(self, req_id: ReqId) -> None: - """Hand a failed attention-only read to scheduler recovery. - - The scheduler's invalid-block recovery currently assumes exactly one - KV cache group. Reporting IDs for a hybrid request would crash it while - unpacking the request's groups, so match NIXL and keep HMA failures out - of this API until scheduler recovery carries group identity. - """ - if self._has_mamba: - return - self._invalid_block_ids.update(self._recving_local_blocks.get(req_id, ())) - - def get_block_ids_with_load_errors(self) -> set[int]: - """Drain the blocks whose read failed, for the scheduler to recompute.""" - invalid, self._invalid_block_ids = self._invalid_block_ids, set() - return invalid - - def save_kv_layer( - self, - metadata: MoRIIOConnectorMetadata, - layer_name: str, - kv_layer: torch.Tensor, - attn_metadata: "AttentionMetadata | None", - **kwargs, - ): - if not self.is_producer: - return - if self.mode == MoRIIOMode.READ: - return - remote_engine_id = None - - for req_id, meta in metadata.reqs_to_save.items(): - # we only need to check if dp0 in rank - remote_engine_id = ( - str(meta.remote_host) + ":" + str(meta.remote_handshake_port) - ) - - meta.remote_engine_id = remote_engine_id - - dp0_remote_engine_id = self.get_engine_name_with_dp(remote_engine_id, 0) - if dp0_remote_engine_id not in self._remote_agents: - # Initiate handshake with remote engine to exchange metadata. - with self._handshake_lock: - if remote_engine_id not in self._remote_agents: - self._background_moriio_handshake( - req_id, remote_engine_id, meta - ) - - continue - self._write_blocks_for_req(req_id, meta, layer_name, kv_layer) - - if remote_engine_id is None: - return - _deadline = time.monotonic() + self.moriio_config.transfer_timeout - while True: - if ( - self._ready_requests.empty() - and remote_engine_id not in self.write_ready_flags - ): - if time.monotonic() > _deadline: - logger.warning( - "Timed out waiting for write_ready_flags[%s]; " - "adjust with kv_connector_extra_config.transfer_timeout", - remote_engine_id, - ) - break - time.sleep(0.001) - continue - elif not self._ready_requests.empty() and ( - remote_engine_id in self.write_ready_flags - ): - self._write_blocks_for_req( - *self._ready_requests.get_nowait(), layer_name, kv_layer - ) - break - else: - break - - def get_engine_name_with_dp(self, engine_name, dp_rank): - return f"{engine_name}_dp{dp_rank}" - - def get_engine_name_with_dp_tp(self, engine_name, dp_rank, tp_rank): - # Per-(dp, tp) session key. The flexible mirror read keys sessions per - # (dp, tp) so one decode worker can hold a session to EACH prefill TP - # rank and spread reads across them; other configs keep the DP-only key. - return f"{engine_name}_dp{dp_rank}_tp{tp_rank}" - - def _eager_handshake_all_dp_ranks(self, metadata: MoRIIOConnectorMetadata) -> None: - """Handshake EVERY remote prefill DP rank BEFORE the decode forward pass, - identically across all local TP workers. - - Why this exists (the deadlock it prevents): with heterogeneous DP prefill - a decode TP worker reads KV from whichever prefill DP rank owns the - request, so across requests every worker must reach several prefill DP - ranks. The decode forward issues per-layer TP collectives (e.g. an - all-gather) that all local TP workers must enter together. If the - handshakes are left to fire lazily on the read path, the workers diverge: - a worker whose target rank is already cached races ahead into the forward - collective while a peer is still blocked in a handshake recv(). The first - worker then waits inside the collective for the stuck peer -> 600s NCCL - timeout / hang. This was observed directly with mixed TP<->DP configs. - - Fix: complete ALL prefill-DP-rank handshakes for every referenced remote - engine HERE, before any read enters the forward, so no worker is still - handshaking once its peers reach a collective. Fires ONCE per remote - engine (first contact), gated by _eager_handshaked_engines. The engine - set comes from scheduler-built metadata (identical on every TP worker), - so all workers run the same handshakes in the same order and reach the - all-reduce barrier below together. - - Failure handling: handshake exceptions are caught, never raised before - the collective (raising early would hang the peers still waiting for it). - Every worker reaches the all-reduce(MIN) vote; if ANY worker failed, ALL - raise the same error AFTER the collective, so the step fails fast and - uniformly in ~seconds instead of one rank hanging the forward for 600s. - """ - import torch.distributed as dist - - # Distinct remote engines referenced this step, in metadata (== - # scheduler) order so every TP worker iterates engines identically. - engines: dict[str, ReqMeta] = {} - for _req_id, meta in metadata.reqs_to_recv.items(): - remote_engine_id = ( - str(meta.remote_host) + ":" + str(meta.remote_handshake_port) - ) - engines.setdefault(remote_engine_id, meta) - - for remote_engine_id, meta in engines.items(): - if remote_engine_id in self._eager_handshaked_engines: - continue - - remote_dp_size = int(meta.remote_dp_size) - port = int(meta.remote_handshake_port) - tp_size = int(meta.tp_size) - - # Flexible mirror (TP prefill + MLA, world_size==1 decode): the read - # round-robins over prefill TP ranks, so pre-warm a session to EVERY - # (dp, tp) rank. Other configs pre-warm per DP rank (tp resolved by - # the fixed local-rank mapping) -- byte-identical to before. The - # mirror's decode is DP+EP, whose forward all-to-all is the collective - # that the eager barrier keeps everyone in step for. - flexible = ( - self.world_size == 1 - and self.use_mla - and remote_dp_size == 1 - and tp_size > 1 - ) - # (engine_id, dp_rank, tp_rank_or_None); tp_rank is None on the legacy - # path so _moriio_handshake falls back to its _remote_tp_rank mapping. - targets: list[tuple[Any, int, int | None]] - if flexible: - targets = [ - (self.get_engine_name_with_dp_tp(remote_engine_id, dp, tp), dp, tp) - for dp in range(remote_dp_size) - for tp in range(max(1, tp_size)) - ] - else: - targets = [ - (self.get_engine_name_with_dp(remote_engine_id, dp), dp, None) - for dp in range(remote_dp_size) - ] - - # Submit handshakes for every not-yet-known target UNDER the lock; do - # NOT hold it across the join or the collective (a stalled recv must - # not block another thread's lock acquisition). Gate on BOTH - # _remote_agents AND layer metadata: a rank with an agent entry but no - # layer metadata is half-handshaked and would KeyError at read time. - futures: list[tuple[str, Future[set[str]]]] = [] - with self._handshake_lock: - for eid, cur_dp_rank, cur_tp_rank in targets: - if ( - eid in self._remote_agents - and eid in self.layer_name_to_remote_kv_cache_metadata - ): - continue - fut = self._handshake_initiation_executor.submit( - self._moriio_handshake, - meta.remote_host, - port, - tp_size, - eid, - cur_dp_rank, - cur_tp_rank, - ) - futures.append((eid, fut)) - - # Join outside the lock. Bounded handshake errors are recorded here - # and reported after the all-reduce. - all_ok = True - results: dict[str, set[str]] = {} - for eid, fut in futures: - try: - results[eid] = fut.result() - except Exception: - logger.exception("Eager MoRIIO handshake failed for %s", eid) - all_ok = False - - with self._handshake_lock: - for eid, agents in results.items(): - self._remote_agents[eid] = agents - - logger.info( - "Eager MoRIIO handshake: engine=%s dp_size=%d new_ranks=%d " - "ok=%s tp_rank=%d", - remote_engine_id, - remote_dp_size, - len(futures), - all_ok, - self.tp_rank, - ) - # CPU all-reduce = TP-uniform success vote AND lockstep barrier: it - # blocks until every TP worker arrives, gives them the same verdict, - # and stays off the model compute stream. - vote = torch.tensor([1 if all_ok else 0], device="cpu", dtype=torch.int32) - dist.all_reduce(vote, group=self.tp_group.cpu_group, op=dist.ReduceOp.MIN) - if int(vote.item()) == 0: - raise HandshakeError( - f"Eager MoRIIO handshake failed for {remote_engine_id} on " - "at least one TP rank; failing this step fast to avoid a " - "TP collective hang" - ) - - self._eager_handshaked_engines.add(remote_engine_id) - - def start_load_kv(self, metadata: MoRIIOConnectorMetadata): - """Start loading by triggering non-blocking moriio_xfer. - We check for these trnxs to complete in each step(). - """ - self.transfer_id_to_request_id = metadata.transfer_id_to_request_id - if self.is_producer: - live_transfer_ids = set(self.transfer_id_to_request_id) - self._consumer_notification_counts = { - transfer_id: count - for transfer_id, count in self._consumer_notification_counts.items() - if transfer_id in live_transfer_ids - } - self._completed_consumer_notifications.intersection_update( - live_transfer_ids - ) - self.moriio_wrapper.async_wait_reqid() - return - if self.mode == MoRIIOMode.WRITE: - return - - # Handshake every referenced remote prefill rank up front, before any - # read enters the forward pass. A lazy per-rank handshake on the read - # path lets TP workers diverge into a forward collective while a peer is - # still blocked handshaking -> NCCL hang (see below). - self._eager_handshake_all_dp_ranks(metadata) - - wait_handshake_readd_req = False - remote_engine_id = None - - for req_id, meta in metadata.reqs_to_recv.items(): - remote_engine_id = ( - str(meta.remote_host) + ":" + str(meta.remote_handshake_port) - ) - meta.remote_engine_id = remote_engine_id - # The eager handshake above already covered every referenced engine - # (and keys the mirror per (dp, tp), which the DP-only dp0 probe below - # would miss). Only fall back to the lazy background handshake for an - # engine it did not cover. - dp0_remote_engine_id = self.get_engine_name_with_dp(remote_engine_id, 0) - if ( - remote_engine_id not in self._eager_handshaked_engines - and dp0_remote_engine_id not in self._remote_agents - ): - # Initiate handshake with remote engine to exchange metadata. - with self._handshake_lock: - if remote_engine_id not in self._remote_agents: - self._background_moriio_handshake( - req_id, remote_engine_id, meta - ) - wait_handshake_readd_req = True - - continue - - # Handshake already completed, start the read xfer. - self._read_blocks_for_req(req_id, meta) - # Start transfers for requests whose handshakes have now finished. - - if remote_engine_id is None and not wait_handshake_readd_req: - self._await_reads_issued_this_step() - return - _deadline = time.monotonic() + self.moriio_config.transfer_timeout - while True: - if ( - self._ready_requests.empty() - and remote_engine_id not in self.load_ready_flag - and wait_handshake_readd_req - ): - if time.monotonic() > _deadline: - logger.warning( - "Timed out waiting for load_ready_flag[%s]; " - "adjust with kv_connector_extra_config.transfer_timeout", - remote_engine_id, - ) - break - time.sleep(0.001) - continue - elif ( - not self._ready_requests.empty() - and remote_engine_id in self.load_ready_flag - ): - self._read_blocks_for_req(*self._ready_requests.get_nowait()) - break - else: - break - - self._await_reads_issued_this_step() - self._reqs_to_send.update(metadata.reqs_to_send) - - def wait_for_save(self, metadata: MoRIIOConnectorMetadata): - if self.mode == MoRIIOMode.WRITE and self.is_producer: - for layer_name, kv_layer in self.kv_caches.items(): - self.save_kv_layer(metadata, layer_name, kv_layer, None) - self._writer.seal_pending_transfers() - - def _next_flex_tp_rank(self, remote_tp_size: int) -> int: - """Deterministic round-robin over prefill tp0..N-1 for the flexible read. - - Round-robin (not random): exactly uniform and testable, with the same - prefill-NIC balancing. Seeded from this decode rank's dp_rank so - concurrent decode DP ranks are phase-staggered -- at a given read index - distinct decode ranks target distinct prefill TP ranks. - """ - rr = getattr(self, "_flex_tp_rr", None) - if rr is None: - rr = int(getattr(self, "dp_rank", 0) or 0) - self._flex_tp_rr = rr + 1 - return rr % remote_tp_size - - def _resolve_read_source(self, meta: ReqMeta) -> tuple[int, bool]: - """Resolve (chosen_tp, flexible) for reading this request's KV. - - Flexible mirror (decode world_size==1 + MLA + pure-TP prefill): MLA - replicates the latent KV across the prefill TP ranks, so any is a valid - source; round-robin across them to spread RDMA/NIC load. Otherwise the - source TP rank is fixed by the local-rank mapping (_remote_tp_rank) -- - forward DP8EP->TP8 -> tp0; symmetric TP -> tp_rank -- byte-identical to - prior behaviour. chosen_tp is the single value threaded into the (dp, tp) - session key, the handshake dial and the notify port, so all three address - the SAME prefill rank (drift -> read one rank but notify another -> the - read rank's prefill buffer is never freed). - """ - remote_tp_size = int(meta.tp_size) - flexible = ( - self.world_size == 1 - and self.use_mla - and int(meta.remote_dp_size) == 1 - and remote_tp_size > 1 - ) - if flexible: - chosen_tp = self._next_flex_tp_rank(remote_tp_size) - else: - chosen_tp = self._remote_tp_rank(remote_tp_size) - return chosen_tp, flexible - - def _read_blocks_for_req(self, req_id: str, meta: ReqMeta): - logger.debug( - "Remote agent %s available, calling _read_blocks for req %s", - meta.remote_engine_id, - req_id, - ) - chosen_tp, flexible = self._resolve_read_source(meta) - self._read_blocks( - request_id=req_id, - transfer_id=meta.transfer_id, - dst_engine_id=meta.remote_engine_id, - local_block_ids=meta.local_block_ids, - remote_block_ids=meta.remote_block_ids, - remote_host=meta.remote_host, - remote_notify_port=meta.remote_notify_port, - remote_tp_size=meta.tp_size, - remote_dp_rank=meta.remote_dp_rank, - chosen_tp=chosen_tp, - flexible=flexible, - ) - - def _write_blocks_for_req(self, req_id: ReqId, meta: ReqMeta, layer_name, kv_layer): - if self._has_mamba: - # Reject before even an attention-layer write can be queued. - self._mamba_tp_ratio(int(meta.tp_size)) - # Compute per-request values to pass through the task (no shared state). - # This avoids race conditions when concurrent requests target different - # decode pods - each WriteTask carries its own routing info. - hosts = ( - list(meta.multi_pod_hosts) if meta.multi_pod_hosts else [meta.remote_host] - ) - dp_local = ( - int(meta.remote_dp_size_local) - if meta.remote_dp_size_local - else int(meta.remote_dp_size) - ) - self.schedule_write_blocks( - request_id=req_id, - transfer_id=meta.transfer_id, - dst_engine_id=meta.remote_engine_id, - local_block_ids=meta.local_block_ids, - remote_block_ids=meta.remote_block_ids, - layer_name=layer_name, - kv_layer=kv_layer, - remote_notify_port=meta.remote_notify_port, - remote_ip=meta.remote_host, - multi_pod_hosts=hosts, - remote_dp_size_local=dp_local, - remote_dp_size=int(meta.remote_dp_size), - remote_tp_size=int(meta.tp_size), - ) - - def merge_contiguous_blocks( - self, - offsets_local: list[int], - offsets_remote: list[int], - sizes: list[int], - assume_sorted: bool = False, - ) -> tuple[list[int], list[int], list[int]]: - n = len(offsets_local) - if n == 0: - return [], [], [] - if not (n == len(offsets_remote) == len(sizes)): - raise ValueError("Input list lengths mismatch") - local_arr = np.fromiter(offsets_local, dtype=np.int64, count=n) - remote_arr = np.fromiter(offsets_remote, dtype=np.int64, count=n) - sizes_arr = np.fromiter(sizes, dtype=np.int64, count=n) - - if assume_sorted: - local_sorted = local_arr - remote_sorted = remote_arr - sizes_sorted = sizes_arr - else: - if np.all(local_arr[:-1] <= local_arr[1:]): - local_sorted = local_arr - remote_sorted = remote_arr - sizes_sorted = sizes_arr - else: - sort_idx = np.argsort(local_arr, kind="stable") - local_sorted = local_arr[sort_idx] - remote_sorted = remote_arr[sort_idx] - sizes_sorted = sizes_arr[sort_idx] - - if n == 1: - return ( - [int(local_sorted[0])], - [int(remote_sorted[0])], - [int(sizes_sorted[0])], - ) - - diff_local = local_sorted[1:] - local_sorted[:-1] - diff_remote = remote_sorted[1:] - remote_sorted[:-1] - prev_size = sizes_sorted[:-1] - - contiguous = (diff_local == prev_size) & (diff_remote == prev_size) - - if not contiguous.any(): - return local_sorted.tolist(), remote_sorted.tolist(), sizes_sorted.tolist() - - if contiguous.all(): - total_size = int(sizes_sorted.sum()) - return [int(local_sorted[0])], [int(remote_sorted[0])], [total_size] - - break_positions = np.flatnonzero(~contiguous) + 1 - segment_starts = np.concatenate(([0], break_positions)) - segment_ends = np.concatenate((break_positions, [n])) - - seg_count = len(segment_starts) - merged_local = [0] * seg_count - merged_remote = [0] * seg_count - merged_sizes = [0] * seg_count - - for si in range(seg_count): - s = segment_starts[si] - e = segment_ends[si] - merged_local[si] = int(local_sorted[s]) - merged_remote[si] = int(remote_sorted[s]) - - merged_sizes[si] = int( - local_sorted[e - 1] + sizes_sorted[e - 1] - local_sorted[s] - ) - - return merged_local, merged_remote, merged_sizes - - def _compute_block_transfer_offsets( - self, - layer_name: str, - local_block_ids: list[int], - remote_block_ids: list[int], - remote_moriio_meta: MoRIIOAgentMetadata, - remote_tp_size: int | None = None, - remote_engine_id: EngineId | None = None, - ) -> tuple[list[int], list[int], list[int]]: - """Compute transfer offsets for block data. - - Args: - layer_name: Name of the layer to transfer - local_block_ids: IDs of local blocks - remote_block_ids: IDs of remote blocks - remote_moriio_meta: Metadata of the remote MoRIIO agent - remote_tp_size: Tensor parallel size of the remote engine. Falls - back to the locally recorded value when None. - remote_engine_id: Id of the remote engine the blocks live on. - - Returns: - Tuple of (local_offsets, remote_offsets, transfer_sizes) - - """ - validate_moriio_heterogeneous_tp_kv_heads( - local_tp_size=self.world_size, - remote_tp_size=( - remote_tp_size - if remote_tp_size and remote_tp_size > 0 - else self.world_size - ), - total_num_kv_heads=self.model_config.get_total_num_kv_heads(), - is_mla=self._is_mla_cache_layer(layer_name), - ) - local, remote, sizes = compute_block_transfer_offsets( - layer_name=layer_name, - kv_cache=self.kv_caches[layer_name], - layer_to_spec=self.layer_to_spec, - local_block_ids=local_block_ids, - remote_block_ids=remote_block_ids, - remote_num_blocks=remote_moriio_meta.num_blocks, - merge_fn=lambda local, remote, sizes: self.merge_contiguous_blocks( - local, remote, sizes, assume_sorted=False - ), - ) - if self.kv_layer_mr_offset: - local_off = self.kv_layer_mr_offset[layer_name] - remote_off = self._remote_layer_mr_offset( - layer_name, remote_moriio_meta, remote_engine_id - ) - local = [x + local_off for x in local] - remote = [x + remote_off for x in remote] - return local, remote, sizes - - def _region_session_indices(self, layer_name: str) -> list[int]: - """Flat session indices for a layer's registered regions. - - Sessions are built one per region in registration order (see - _get_built_session), so an attention layer maps to a single index and a - KDA layer to two indices ([conv, ssm]). Built once and cached. - """ - if self._region_session_index is None: - mapping: dict[str, list[int]] = {} - idx = 0 - for ln, metas in self.layer_name_to_local_kv_cache_metadata.items(): - mapping[ln] = list(range(idx, idx + len(metas))) - idx += len(metas) - self._region_session_index = mapping - return self._region_session_index[layer_name] - - def _mamba_tp_ratio(self, remote_tp_size: int | None) -> int: - """Validate the homogeneous-TP KDA contract.""" - remote = self.world_size if remote_tp_size is None else remote_tp_size - if remote != self.world_size: - raise NotImplementedError( - "heterogeneous-TP mamba/KDA transfer is not supported yet " - f"(local_tp={self.world_size}, remote_tp={remote}); use equal " - "prefill/decode TP for hybrid models" - ) - return 1 - - def _mamba_blocks_for_layer( - self, - layer_name: str, - block_groups: BlockIds, - ) -> list[int]: - payload_index = self._mamba_payload_index_by_layer.get(layer_name) - if payload_index is None or payload_index >= len(block_groups): - raise MoRIIOError( - f"KDA layer {layer_name} has no block group in the transfer payload" - ) - return list(block_groups[payload_index]) - - def _compute_mamba_transfer_offsets( - self, - layer_name: str, - local_slots: list[int], - remote_slots: list[int], - remote_tp_size: int | None = None, - ) -> tuple[list[int], list[int], list[int], int]: - """Compute (local, remote, sizes, n_conv) for a KDA layer. - - Entries [:n_conv] address the conv region/session; entries [n_conv:] - address the ssm region/session. - """ - assert self._conv_decomp is not None, "KDA decomposition not derived" - conv, ssm = kda_conv_ssm( - self.kv_caches[layer_name], - cast(MambaSpec, self.layer_to_spec[layer_name]), - ) - tp_ratio = self._mamba_tp_ratio(remote_tp_size) - # Cache the slot-independent geometry per layer; only the per-request - # slot bases vary, and heterogeneous TP is rejected above. - template = self._mamba_offset_templates.get(layer_name) - if template is None: - template = build_mamba_offset_template( - conv, - ssm, - self._conv_decomp, - tp_ratio, - ) - self._mamba_offset_templates[layer_name] = template - local_offs, remote_offs, sizes = apply_mamba_offset_template( - template, local_slots, remote_slots - ) - n_conv = compute_mamba_conv_split_count(local_slots, self._conv_decomp) - return local_offs, remote_offs, sizes, n_conv - - def _post_mamba_reads( - self, - layer_name: str, - sessions: list, - region_sessions: list[int], - local_slots: list[int], - remote_slots: list[int], - remote_tp_size: int, - request_id: str, - deadline: float, - ) -> list: - """Post one KDA layer's conv and ssm reads; return their statuses. - - The layer owns two registered regions, so the offset list is split at - the conv/ssm boundary and each half goes to its own session. A half - with nothing to move is skipped rather than posted empty. - """ - local, remote, sizes, n_conv = self._compute_mamba_transfer_offsets( - layer_name, - local_slots, - remote_slots, - remote_tp_size=remote_tp_size, - ) - statuses: list = [] - for sess_idx, sl in ( - (region_sessions[0], slice(0, n_conv)), - (region_sessions[1], slice(n_conv, None)), - ): - region_sizes = sizes[sl] - if not region_sizes: - continue - statuses.append( - self._post_read_with_backoff( - sessions[sess_idx], - region_sizes, - local[sl], - remote[sl], - request_id, - layer_name, - deadline, - ) - ) - return statuses - - def _post_read_with_backoff( - self, - session, - sizes: list[int], - local_offsets: list[int], - remote_offsets: list[int], - request_id: str, - layer_name: str, - deadline: float, - ): - """Post one RDMA READ, backing off on transient SQ-full rejection. - - read_remote_data posts synchronously, so a send-queue-full rejection is - a Failed() status on return; a separate CQ-poll thread drains - completions and frees SQ depth, so we back off and re-post until - transfer_timeout, then store the failed status (get_finished notifies - prefill and drops the request non-fatally). - """ - _backoff = _SQ_FULL_BACKOFF_INITIAL_S - while True: - transfer_status = self.moriio_wrapper.read_remote_data( - sizes, local_offsets, remote_offsets, session - ) - if not self._is_sq_full_status(transfer_status): - break - if time.monotonic() > deadline: - logger.warning( - "MoRIIO READ send queue stayed full past " - "transfer_timeout for req %s layer %s; storing failed " - "status (get_finished notifies prefill and drops the " - "request). Raise qp_per_transfer if frequent.", - request_id, - layer_name, - ) - break - time.sleep(_backoff) - _backoff = min(_backoff * 2, _SQ_FULL_BACKOFF_MAX_S) - return transfer_status - - @staticmethod - def _is_sq_full_status(status) -> bool: - """True if a MoRIIO transfer status is a transient RDMA send-queue-full - rejection (retryable backpressure), not a terminal failure. - - read_remote_data posts the RDMA READ synchronously (the mori executor - joins its worker before returning and marks the status on the calling - thread), so a send-queue-full rejection is a Failed() status the moment - the call returns. mori surfaces it as a generic ERR_RDMA_OP carrying - "SQ full" in the message (no distinct code), so we match the message. - Only meaningful once status.Failed() is True. - """ - try: - return bool(status.Failed()) and "SQ full" in (status.Message() or "") - except Exception: - return False - - def _read_blocks( - self, - local_block_ids: BlockIds, - remote_block_ids: BlockIds, - dst_engine_id: str, - request_id: str, - transfer_id: str, - remote_host: str, - remote_notify_port: int, - remote_tp_size: int, - remote_dp_rank: int = 0, - chosen_tp: int | None = None, - flexible: bool = False, - ) -> None: - if self.mode == MoRIIOMode.WRITE: - return - - # Read from the prefill rank that actually computed this request's KV - # (forwarded by the proxy). Hardcoding DP0 reads from a different rank's - # memory registration; per-rank num_blocks differ, so high block ids can - # overrun the wrong rank's region. - # - # eff_tp = the remote TP rank this read targets. The flexible mirror - # reads from a round-robin-chosen prefill TP rank and keys the session - # per (dp, tp); other configs use the fixed local-rank mapping (eff_tp == - # _remote_tp_rank), byte-identical to before. This key MUST match the one - # the eager handshake stored the session under. - eff_tp = ( - int(chosen_tp) - if chosen_tp is not None - else self._remote_tp_rank(remote_tp_size) - ) - if not any(local_block_ids): - # A request aborted before decode allocation still carries the - # producer's block table. There is no local destination or forward, - # so release the producer blocks without building sessions or - # posting RDMA reads. - self.moriio_wrapper.send_notify( - transfer_id, - remote_host, - str( - remote_notify_port - + get_port_offset( - int(remote_dp_rank), - eff_tp, - remote_tp_size, - ) - ), - message_type="release", - message_fields={"consumer_tp_size": self.world_size}, - ) - return - - if self._has_mamba: - expected_groups = 1 + self._num_mamba_transfer_groups - if ( - len(local_block_ids) != expected_groups - or len(remote_block_ids) != expected_groups - ): - raise MoRIIOError( - "MoRIIO hybrid READ block group count mismatch: " - f"expected={expected_groups}, local={len(local_block_ids)}, " - f"remote={len(remote_block_ids)}" - ) - local_attn = list(local_block_ids[0]) - remote_attn = list(remote_block_ids[0]) - self._mamba_tp_ratio(remote_tp_size) - else: - local_attn = remote_attn = [] - - if flexible: - remote_dp_engine_id = self.get_engine_name_with_dp_tp( - dst_engine_id, int(remote_dp_rank), eff_tp - ) - else: - remote_dp_engine_id = self.get_engine_name_with_dp( - dst_engine_id, int(remote_dp_rank) - ) - sessions, remote_moriio_meta = self._get_built_session(remote_dp_engine_id) - - # SQ-full backpressure deadline, shared across this request's layers. - _sq_deadline = time.monotonic() + self.moriio_config.transfer_timeout - # TODO: apply multi-session batch-read when MoRIIO supports it. - for layer_name in self.layer_name_to_local_kv_cache_metadata: - region_sessions = self._region_session_indices(layer_name) - statuses = [] - if self._is_mamba_layer(layer_name): - local_mamba = self._mamba_blocks_for_layer(layer_name, local_block_ids) - remote_mamba = self._mamba_blocks_for_layer( - layer_name, remote_block_ids - ) - # KDA layer: pull conv (sub-projection offsets) and ssm at the - # request's recurrent-state slot. Two regions -> two sessions. - if not local_mamba or not remote_mamba: - if not local_attn and not remote_attn: - # Whole-request no-op (e.g. full prefix cache hit): - # nothing to transfer for any layer. - continue - # A KDA layer must have its recurrent-state slot in the - # block-id tuple; missing it (while attention blocks are - # present) means the mamba KV-cache group is absent -> bug. - raise MoRIIOError( - f"KDA layer {layer_name}: missing mamba recurrent-state " - f"block ids (local={local_mamba}, remote={remote_mamba}) " - f"with attention blocks present for request {request_id}; " - "the mamba KV-cache group is absent from the transfer " - "block-id tuple" - ) - statuses = self._post_mamba_reads( - layer_name, - sessions, - region_sessions, - local_mamba, - remote_mamba, - remote_tp_size, - request_id, - _sq_deadline, - ) - else: - if self._has_mamba: - local_layer_blocks = local_attn - remote_layer_blocks = remote_attn - else: - group_idx = self.layer_to_group[layer_name] - local_layer_blocks = local_block_ids[group_idx] - remote_layer_blocks = remote_block_ids[group_idx] - offs = self._compute_block_transfer_offsets( - layer_name, - local_layer_blocks, - remote_layer_blocks, - remote_moriio_meta, - remote_tp_size=remote_tp_size, - remote_engine_id=remote_dp_engine_id, - ) - statuses.append( - self._post_read_with_backoff( - sessions[region_sessions[0]], - offs[2], - offs[0], - offs[1], - request_id, - layer_name, - _sq_deadline, - ) - ) - with self.moriio_wrapper.lock: - self._recving_transfers[request_id][layer_name] = statuses - self._recving_transfers_start.setdefault(request_id, time.monotonic()) - # Destination blocks, kept for _record_failed_recv. - if self._has_mamba: - failed_recv_blocks = local_attn - else: - failed_recv_blocks = [ - block_id for group in local_block_ids for block_id in group - ] - self._recving_local_blocks.setdefault( - request_id, list(failed_recv_blocks) - ) - # Awaited before the forward when the per-layer barrier cannot - # run (see _await_reads_issued_this_step). - self._reads_issued_this_step.extend(statuses) - if self._is_mamba_layer(layer_name): - # No per-layer barrier exists for mamba layers, so these are - # always awaited in start_load_kv. - self._mamba_reads_this_step.extend(statuses) - self._recving_transfers_callback_addr[request_id] = ( - remote_host, - str( - remote_notify_port - + get_port_offset( - int(remote_dp_rank), - eff_tp, - remote_tp_size, - ) - ), - transfer_id, - ) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_engine.py b/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_engine.py deleted file mode 100644 index 3ffd6d10f9..0000000000 --- a/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_engine.py +++ /dev/null @@ -1,1084 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -import threading -import time -from collections import OrderedDict, defaultdict -from math import ceil -from queue import Empty, Queue -from typing import TYPE_CHECKING, Any -from weakref import ref as weakref_ref - -import msgpack -import torch -import zmq - -from vllm.logger import init_logger -from vllm.utils.network_utils import ( - make_zmq_path, - make_zmq_socket, -) - -if TYPE_CHECKING: - from mori.io import BackendType - -from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_common import ( - ROLE, - HandshakeError, - LayerTransferPlan, - MoRIIOAgentMetadata, - MoRIIOConstants, - MoRIIOError, - MoRIIOTransferAck, - RemoteAllocInfo, - TransferBatchState, - TransferError, - TransferId, - WriteTask, - fold_local_rank, - get_port_offset, - get_role, - pod_index, - zmq_ctx, -) - -if TYPE_CHECKING: - from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_connector import ( - MoRIIOConnectorWorker, - ) - -logger = init_logger(__name__) -try: - from mori.io import ( - BackendType, - EngineDesc, - IOEngine, - MemoryDesc, - PollCqMode, - RdmaBackendConfig, - XgmiBackendConfig, - ) - - logger.info("MoRIIO is available") -except ImportError: - logger.error("MoRIIO is not available") - -try: - from mori.io import StatusCode -except ImportError: - StatusCode = None - -"""Write task execution logic for MoRIIO connector.""" - - -_MAX_TERMINAL_TRANSFER_IDS = 4096 - - -WriteGeometryKey = tuple[tuple[int, ...], tuple[int, ...], torch.dtype] - - -def _get_write_geometry_key(kv_cache: torch.Tensor) -> WriteGeometryKey: - return (tuple(kv_cache.shape), tuple(kv_cache.stride()), kv_cache.dtype) - - -class MoRIIOWriter: - """Handles write operations for KV cache transfers. - - WRITE mode state machine: - D sends destination block allocation, P schedules one write per layer - after the layer CUDA event, P seals the scheduled write count after - forward, then P notifies D and releases P blocks after all scheduled - writes complete. - """ - - def __init__(self, worker: "MoRIIOConnectorWorker"): - """Initialize the writer. - - Args: - worker: Reference to the parent worker - - """ - self._worker_ref: weakref_ref[MoRIIOConnectorWorker] = weakref_ref(worker) - self._write_task_q: Queue[WriteTask] = Queue() - self._write_worker_started = False - self._write_worker_lock = threading.Lock() - self._write_state_lock = threading.Lock() - self._deferred_tasks: list[WriteTask] = [] - self._scheduled_writes: dict[TransferId, int] = defaultdict(int) - self._scheduled_layers: dict[TransferId, set[str]] = defaultdict(set) - self._sealed_writes: dict[TransferId, int] = {} - self._defer_timeout = worker.moriio_config.defer_timeout - - @property - def worker(self) -> "MoRIIOConnectorWorker": - """Get the worker instance. - - Returns: - The parent worker instance - - Raises: - RuntimeError: If worker has been garbage collected - - """ - worker = self._worker_ref() - if worker is None: - raise RuntimeError("Parent worker has been garbage collected") - return worker - - def ensure_worker_started(self) -> None: - """Ensure the background write worker is running.""" - if self._write_worker_started: - return - self._write_worker_started = True - with self._write_worker_lock: - thread = threading.Thread( - target=self._write_worker_loop, daemon=True, name="moriio-write-worker" - ) - thread.start() - logger.info("Started MoRIIO write worker thread") - - def schedule_write(self, task: WriteTask) -> bool: - """Schedule a write task. - - Args: - task: The write task to schedule - - """ - self.ensure_worker_started() - if self._is_transfer_terminal(task.transfer_id): - return False - - with self._write_state_lock: - if self._is_transfer_terminal(task.transfer_id): - return False - if task.layer_name in self._scheduled_layers[task.transfer_id]: - return False - self._scheduled_layers[task.transfer_id].add(task.layer_name) - self._scheduled_writes[task.transfer_id] += 1 - self._write_task_q.put(task) - return True - - def is_scheduled(self, transfer_id: TransferId, layer_name: str) -> bool: - with self._write_state_lock: - return layer_name in self._scheduled_layers.get(transfer_id, set()) - - def seal_pending_transfers(self) -> None: - """Seal expected WRITE counts after the model forward has run. - - `save_kv_layer` is only invoked for attention layers whose backend uses - the standard KV connector hook. Hybrid models can register more KV - cache tensors than the number of hooks that fire in a forward, so WRITE - completion must be based on the tasks actually queued for the transfer. - """ - pending: list[tuple[TransferId, RemoteAllocInfo]] = [] - with self._write_state_lock: - for transfer_id, write_count in self._scheduled_writes.items(): - if transfer_id in self._sealed_writes: - continue - self._sealed_writes[transfer_id] = write_count - request_info = ( - self.worker.moriio_wrapper.done_remote_allocate_req_dict.get( - transfer_id - ) - ) - if request_info is not None: - request_info.writes_expected = write_count - pending.append((transfer_id, request_info)) - - for transfer_id, request_info in pending: - self._finalize_if_complete(transfer_id, request_info) - - def _write_worker_loop(self) -> None: - """Main loop for the write worker thread.""" - while True: - # Process deferred tasks first - self._process_deferred_tasks() - - # Get new task - try: - task = self._write_task_q.get(timeout=0.01) - except Empty: - continue - - if self._is_transfer_terminal(task.transfer_id): - continue - - # Check if remote blocks are ready - if not self._is_remote_ready(task): - # task.retry_count += 1 - self._deferred_tasks.append(task) - # logger.debug( - # "Deferred task for request %s (retry %d)", - # task.request_id, task.retry_count - # ) - continue - - # Execute the task - try: - self._execute_write_task(task) - except Exception: - logger.exception( - "Write task failed for request %s, marking done", - task.request_id, - ) - self._mark_request_done(task.transfer_id) - - def _process_deferred_tasks(self) -> None: - """Process tasks that were previously deferred.""" - if not self._deferred_tasks: - return - - defer_timeout = self._defer_timeout - now = time.perf_counter() - still_deferred: list[WriteTask] = [] - - for task in self._deferred_tasks: - if self._is_transfer_terminal(task.transfer_id): - continue - if self._is_remote_ready(task): - try: - self._execute_write_task(task) - except Exception: - logger.exception( - "Deferred write task failed for request %s, marking done", - task.request_id, - ) - self._mark_request_done(task.transfer_id) - elif ( - now - task.enqueue_time > defer_timeout - and self.worker.moriio_wrapper.fail_unallocated_write(task.transfer_id) - ): - self._notify_write_failed(task, now - task.enqueue_time) - else: - still_deferred.append(task) - - self._deferred_tasks = still_deferred - - def _notify_write_failed(self, task: WriteTask, age: float) -> None: - self._clear_transfer_state(task.transfer_id) - logger.error( - "Deferred write task for request %s timed out after %.1fs waiting " - "for remote blocks", - task.request_id, - age, - ) - # No remote allocation arrived, so the decode DP rank that owns the - # request is unknown. Tell every rank; the others drop it. - for decode_dp_rank in range(max(task.remote_dp_size, 1)): - remote_ip, remote_port = self._resolve_notify_endpoint(task, decode_dp_rank) - try: - self.worker.moriio_wrapper.send_notify( - task.transfer_id, - remote_ip, - remote_port, - message_type="write_failed", - ) - except Exception: - logger.exception( - "Failed to notify decode DP rank %d that transfer %s timed out", - decode_dp_rank, - task.transfer_id, - ) - - def _clear_transfer_state(self, transfer_id: TransferId) -> None: - with self._write_state_lock: - self._scheduled_writes.pop(transfer_id, None) - self._scheduled_layers.pop(transfer_id, None) - self._sealed_writes.pop(transfer_id, None) - - def _is_transfer_terminal(self, transfer_id: TransferId) -> bool: - wrapper = self.worker.moriio_wrapper - with wrapper.lock: - return wrapper._is_transfer_terminal_locked(transfer_id) - - def _mark_request_done(self, transfer_id: str) -> None: - """Mark a request done so its blocks are freed, even on transfer failure.""" - wrapper = self.worker.moriio_wrapper - with wrapper.lock: - wrapper.done_req_ids.append(MoRIIOTransferAck(transfer_id)) - wrapper.done_remote_allocate_req_dict.pop(transfer_id, None) - wrapper._mark_transfer_terminal_locked(transfer_id) - self._clear_transfer_state(transfer_id) - - def _is_remote_ready(self, task: WriteTask) -> bool: - """Check if remote blocks are allocated for this task. - - Args: - task: The write task - - Returns: - True if remote blocks are ready - - """ - return ( - task.transfer_id in self.worker.moriio_wrapper.done_remote_allocate_req_dict - ) - - def _get_remote_alloc_info(self, transfer_id: str) -> RemoteAllocInfo: - """Get remote allocation info for a request. - - Args: - transfer_id:TransferId The request ID - - Returns: - Remote allocation information - - Raises: - KeyError: If allocation info is missing - - """ - try: - return self.worker.moriio_wrapper.done_remote_allocate_req_dict[transfer_id] - except KeyError as e: - raise KeyError( - f"Remote allocation info missing for transfer {transfer_id}" - ) from e - - def _execute_write_task(self, task: WriteTask) -> None: - """Execute a single write task. - - Args: - task: The write task to execute - - """ - # Get remote allocation info - request_info = self._get_remote_alloc_info(task.transfer_id) - with self._write_state_lock: - request_info.completion_request_id = task.request_id - # Resolve the final notify endpoint under the lock so finalize only - # reads it. - ( - request_info.completion_remote_ip, - request_info.completion_notify_port, - ) = self._resolve_notify_endpoint(task, request_info.decode_dp_rank) - if task.transfer_id in self._sealed_writes: - request_info.writes_expected = self._sealed_writes[task.transfer_id] - - if request_info.block_ids is None: - logger.debug( - "Request remote block IDs not ready:request_id = %s, transfer_id = %s", - task.request_id, - task.transfer_id, - ) - return - - # Wait for CUDA event - # The attention computation of the current layer cannot - # overlap with the kv transfer task, - # otherwise it will cause precision issues. - # This event is used to synchronize the kv transfer and computation tasks. - task.event.synchronize() - - # Update engine ID with DP rank - task.dst_engine_id = self.worker.get_engine_name_with_dp( - task.dst_engine_id, request_info.decode_dp_rank - ) - - # Get or create sessions - sessions, remote_moriio_meta = self.worker._get_built_session( - task.dst_engine_id - ) - - if self.worker._is_mamba_layer(task.layer_name): - plans = self._prepare_mamba_transfer_plans(task, request_info) - else: - plans = [ - self._prepare_transfer_plan(task, request_info, remote_moriio_meta) - ] - transfer_statuses = [] - for plan in plans: - transfer_statuses.extend(self._do_layer_write(plan, sessions)) - with self._write_state_lock: - request_info.transfer_statuses.extend(transfer_statuses) - - # Finalize if all layers complete - self._mark_write_done(task.transfer_id, request_info) - - def _prepare_transfer_plan( - self, - task: WriteTask, - request_info: RemoteAllocInfo, - remote_moriio_meta: MoRIIOAgentMetadata, - ) -> LayerTransferPlan: - """Prepare the transfer plan for a layer. - - Args: - task: The write task - request_info: Remote allocation information - remote_moriio_meta: Metadata of the remote MoRIIO agent - - Returns: - The transfer plan - - """ - layer_cache = self.worker.kv_caches[task.layer_name] - key = (task.layer_name, *_get_write_geometry_key(layer_cache)) - offsets = request_info.transfer_offsets.get(key) - if offsets is None: - group = self.worker.layer_to_group[task.layer_name] - assert request_info.block_ids is not None - local_blocks = task.local_block_ids[group] - remote_blocks = request_info.block_ids[group] - if self.worker._has_mamba: - # D can reserve a tail block for the token recomputed locally. - remote_blocks = remote_blocks[: len(local_blocks)] - offsets = self.worker._compute_block_transfer_offsets( - task.layer_name, - local_blocks, - remote_blocks, - remote_moriio_meta, - remote_engine_id=task.dst_engine_id, - ) - request_info.transfer_offsets[key] = offsets - - # One session per registered region; attention layers map to a single - # index (see MoRIIOConnectorWorker._region_session_indices). - sess_idx = self.worker._region_session_indices(task.layer_name)[0] - - local_off, remote_off, sizes = offsets - - return LayerTransferPlan( - request_id=task.request_id, - transfer_id=task.transfer_id, - layer_name=task.layer_name, - sess_idx=sess_idx, - transfer_local_offsets=local_off, - transfer_remote_offsets=remote_off, - transfer_sizes=sizes, - use_batch=True, - ) - - def _prepare_mamba_transfer_plans( - self, task: WriteTask, request_info: RemoteAllocInfo - ) -> list[LayerTransferPlan]: - """Plan both registered regions as one scheduled layer write.""" - assert request_info.block_ids is not None - local_slots = self.worker._mamba_blocks_for_layer( - task.layer_name, task.local_block_ids - ) - remote_slots = self.worker._mamba_blocks_for_layer( - task.layer_name, request_info.block_ids - ) - if len(local_slots) != 1 or len(remote_slots) != 1: - raise MoRIIOError( - "Hybrid WRITE requires one running-state slot per Mamba group" - ) - local, remote, sizes, n_conv = self.worker._compute_mamba_transfer_offsets( - task.layer_name, - local_slots, - remote_slots, - remote_tp_size=task.remote_tp_size, - ) - conv_session, ssm_session = self.worker._region_session_indices(task.layer_name) - return [ - LayerTransferPlan( - request_id=task.request_id, - transfer_id=task.transfer_id, - layer_name=task.layer_name, - sess_idx=session, - transfer_local_offsets=local[region], - transfer_remote_offsets=remote[region], - transfer_sizes=sizes[region], - ) - for session, region in ( - (conv_session, slice(0, n_conv)), - (ssm_session, slice(n_conv, None)), - ) - ] - - def _do_layer_write(self, plan: LayerTransferPlan, sessions: list) -> list[Any]: - """Perform the actual layer write. - - Args: - plan: The transfer plan - sessions: List of transfer sessions - - """ - if not plan.transfer_sizes: - return [] - if plan.use_batch: - return [ - self.worker.moriio_wrapper.write_remote_data( - plan.transfer_sizes, - plan.transfer_local_offsets, - plan.transfer_remote_offsets, - sessions[plan.sess_idx], - ) - ] - - transfer_statuses: list[Any] = [] - for i in range(len(plan.transfer_local_offsets)): - transfer_statuses.append( - self.worker.moriio_wrapper.write_remote_data_single( - plan.transfer_sizes[i], - plan.transfer_local_offsets[i], - plan.transfer_remote_offsets[i], - plan.sess_idx, - ) - ) - return transfer_statuses - - def _mark_write_done( - self, transfer_id: TransferId, request_info: RemoteAllocInfo - ) -> None: - """Record one completed WRITE task and finalize if sealed.""" - with self._write_state_lock: - request_info.writes_done += 1 - self._finalize_if_complete(transfer_id, request_info) - - def _finalize_if_complete( - self, transfer_id: TransferId, request_info: RemoteAllocInfo - ) -> None: - """Finalize transfer if all scheduled writes are complete.""" - with self._write_state_lock: - expected = request_info.writes_expected - if expected is None or request_info.writes_done < expected: - return - if request_info.completion_notified: - return - request_id = request_info.completion_request_id - remote_port = request_info.completion_notify_port - remote_ip = request_info.completion_remote_ip - if request_id is None or remote_port is None or remote_ip is None: - return - transfer_statuses = list(request_info.transfer_statuses) - request_info.transfer_statuses.clear() - request_info.completion_notified = True - - # Wait for this request's transfers to complete. - self.worker.moriio_wrapper.waiting_for_transfer_complete(transfer_statuses) - - # Consider using RDMA immediate data in decode side - # to eliminate the need for this notification. - # Consider including the first gen token from prefill in the notification - - # Send completion notification - self.worker.moriio_wrapper.send_notify( - transfer_id, remote_ip, remote_port, message_type="write_done" - ) - # mark request as done, then we can free the blocks - with self.worker.moriio_wrapper.lock: - self.worker.moriio_wrapper.done_req_ids.append( - MoRIIOTransferAck(transfer_id) - ) - self.worker.moriio_wrapper.done_remote_allocate_req_dict.pop( - transfer_id, None - ) - self.worker.moriio_wrapper._mark_transfer_terminal_locked(transfer_id) - self._clear_transfer_state(transfer_id) - logger.debug( - "Completed transfer for (request, transfer) %s, %s, notified port %d", - request_id, - transfer_id, - remote_port, - ) - - def _resolve_notify_endpoint( - self, task: WriteTask, decode_dp_rank: int - ) -> tuple[str, int]: - # Wide-EP multi-pod: task.remote_ip addresses only the first pod, so - # resolve the per-rank host from multi_pod_hosts (falls back to - # task.remote_ip for single-pod or on any indexing miss). The port - # offset uses the per-pod local rank, since each pod binds notify - # sockets only for its local ranks. - decode_dp_rank = int(decode_dp_rank) - dp_local = task.remote_dp_size_local - remote_ip = task.remote_ip - if task.multi_pod_hosts and dp_local > 0: - remote_pod = pod_index(decode_dp_rank, dp_local) - if 0 <= remote_pod < len(task.multi_pod_hosts): - remote_ip = task.multi_pod_hosts[remote_pod] - remote_port = task.remote_notify_port + get_port_offset( - fold_local_rank(decode_dp_rank, dp_local), self.worker.tp_rank - ) - return remote_ip, remote_port - - -class MoRIIOWrapper: - """Wrapper for MoRIIO engine operations. - - Handles both producer and consumer roles for KV cache transfers. - - Args: - moriio_engine: MoRIIO engine instance - tp_rank: Tensor parallel rank - dp_rank: Data parallel rank - - """ - - def __init__( - self, - moriio_engine: "IOEngine | None" = None, - tp_rank: int = 0, - dp_rank: int = 0, - transfer_timeout: float = MoRIIOConstants.DEFAULT_TRANSFER_TIMEOUT, - ): - self.tp_rank = tp_rank - self.dp_rank = dp_rank - self.moriio_engine = moriio_engine - self.remote_memory_metadata = None - self.local_memory_registered = False - self.local_memory_metadata = None - self.transfer_status: list[Any] = [] - self.remote_engine_ip: str | None = None - self.notify_port: int | None = None - self.lock = threading.Lock() - self.done_req_ids: list[MoRIIOTransferAck] = [] - self.done_remote_allocate_req_dict: dict[TransferId, RemoteAllocInfo] = {} - self.done_write_cache_req_ids: list[str] = [] - self.failed_write_cache_req_ids: list[str] = [] - self._terminal_transfer_ids: OrderedDict[TransferId, None] = OrderedDict() - self._transfer_timeout = transfer_timeout - self.notify_thread: threading.Thread | None = None - self.sessions: list[IOEngine.Session] = [] - self.paths: dict[str, zmq.Socket] = {} - - def set_moriio_engine(self, moriio_engine): - assert moriio_engine is not None, ( - "You Cannot pass None engine to MoRIIOWrapper!" - ) - self.moriio_engine = moriio_engine - - def set_backend_type( - self, - backend_type: "BackendType", - qp_per_transfer: int = 1, - post_batch_size: int = -1, - num_workers: int = 1, - ) -> None: - assert self.moriio_engine is not None, "MoRIIO engine must be set first" - if backend_type == BackendType.XGMI: - logger.info("Using MoRIIO backend: XGMI") - self.moriio_engine.create_backend(backend_type, XgmiBackendConfig()) - else: - logger.info( - "Using MoRIIO backend: RDMA " - "(qp_per_transfer=%d, post_batch_size=%d, num_workers=%d)", - qp_per_transfer, - post_batch_size, - num_workers, - ) - rdma_cfg = RdmaBackendConfig( - qp_per_transfer, - post_batch_size, - num_workers, - PollCqMode.POLLING, - # vLLM uses ZMQ for completion signaling - # and never calls PopInboundTransferStatus. - # With notifications enabled, ibv_post_send - # ENOMEM at high concurrency permanently - # poisons TransferStatus before the data WR - # completes, hanging requests in - # WAITING_FOR_REMOTE_KVS indefinitely. - enable_notification=False, - ) - self.moriio_engine.create_backend(backend_type, rdma_cfg) - - def get_agent_metadata(self): - assert self.moriio_engine is not None, "MoRIIO engine must be set first" - engine_metadata = self.moriio_engine.get_engine_desc() - engine_metadata_packed = engine_metadata.pack() - return engine_metadata_packed - - def register_remote_engine(self, remote_packed_engine_metadata): - assert self.moriio_engine is not None, "MoRIIO engine must be set first" - consumer_engine_metadata = EngineDesc.unpack(remote_packed_engine_metadata) - self.moriio_engine.register_remote_engine(consumer_engine_metadata) - return consumer_engine_metadata.key - - def register_local_tensor(self, tensor: torch.Tensor): - assert self.moriio_engine is not None, "MoRIIO engine must be set first" - try: - self.local_memory_metadata = self.moriio_engine.register_torch_tensor( - tensor - ) - assert self.local_memory_metadata is not None, ( - "register_torch_tensor returned None" - ) - local_memory_metadata_packed = self.local_memory_metadata.pack() - except Exception as e: - raise MoRIIOError(f"Failed to register local memory: {e}") from e - self.local_memory_registered = True - return local_memory_metadata_packed - - def get_unpack_memory_metadata(self, packed_memory_metadata): - return MemoryDesc.unpack(packed_memory_metadata) - - def build_session(self, local_memory_metadata, remote_memory_metadata): - assert self.moriio_engine is not None, "MoRIIO engine must be set first" - return self.moriio_engine.create_session( - local_memory_metadata, remote_memory_metadata - ) - - def read_remote_data( - self, transfer_size_byte, local_offset=0, remote_offset=0, session=None - ): - assert self.local_memory_registered, "You have not register local memory data!" - assert self.moriio_engine is not None, "MoRIIO engine must be set first" - transfer_status = session.batch_read( - local_offset, - remote_offset, - transfer_size_byte, - self.moriio_engine.allocate_transfer_uid(), - ) - - return transfer_status - - def write_remote_data( - self, transfer_size_byte, local_offset=0, remote_offset=0, session=None - ): - assert self.local_memory_registered, "You have not register local memory data!" - assert self.moriio_engine is not None, "MoRIIO engine must be set first" - write_uid = self.moriio_engine.allocate_transfer_uid() - - transfer_status = session.batch_write( - local_offset, remote_offset, transfer_size_byte, write_uid - ) - return transfer_status - - def write_remote_data_single( - self, transfer_size_byte, local_offset=0, remote_offset=0, sess_idx=0 - ): - assert self.local_memory_registered, "You have not register local memory data!" - assert self.moriio_engine is not None, "MoRIIO engine must be set first" - transfer_status = self.sessions[sess_idx].write( - local_offset, - remote_offset, - transfer_size_byte, - self.moriio_engine.allocate_transfer_uid(), - ) - return transfer_status - - def poll_transfer_batch(self, transfer_statuses: list[Any]) -> TransferBatchState: - """Non-blocking verdict over every status of one request. - - Judging a request by its newest status alone calls it done while an - earlier layer is unfinished or already failed; failures do land on - layers other than the last, because _post_read_with_backoff returns a - failed status when the send queue never drains. - - This stays a Python scan so it does not drive the backend's progress - callback from the engine thread alongside MoRIIO's own poller. - """ - state = TransferBatchState.DONE - for status in transfer_statuses: - if status.Failed(): - return TransferBatchState.FAILED - if not status.Succeeded(): - state = TransferBatchState.PENDING - return state - - def waiting_for_transfer_complete(self, transfer_statuses: list[Any] | None = None): - if transfer_statuses is None: - with self.lock: - transfers_to_wait = self.transfer_status[:] - self.transfer_status.clear() - else: - transfers_to_wait = list(transfer_statuses) - - if not transfers_to_wait: - return - - wait_all = getattr(self.moriio_engine, "wait_all", None) - if wait_all is None or StatusCode is None: - self._poll_transfers_until_done(transfers_to_wait) - return - - # Zero means a nonblocking progress call in Mori, not an immediate - # deadline. Round up so even sub-millisecond waits remain blocking. - timeout_ms = ceil(self._transfer_timeout * 1000) - result = wait_all(transfers_to_wait, timeout_ms=timeout_ms) - if result != StatusCode.SUCCESS: - details = [ - f"{status.Message()} (code={status.Code()})" - for status in transfers_to_wait - if status.Failed() - ] - if not details and result in (StatusCode.INIT, StatusCode.IN_PROGRESS): - details.append( - f"RDMA transfer timed out after {self._transfer_timeout:g}s" - ) - raise TransferError( - f"MoRIIO batch wait failed ({result}): " + "; ".join(details) - ) - - def _poll_transfers_until_done(self, transfers_to_wait: list[Any]) -> None: - """Fallback for mori builds without the batched wait. - - Gives failure precedence over pending: once any status has failed the - request is lost, so waiting out the remaining ones would only delay the - barrier by up to transfer_timeout. - """ - timeout = self._transfer_timeout - deadline = time.monotonic() + timeout - remaining = list(transfers_to_wait) - errors: list[str] = [] - - while remaining: - timed_out = time.monotonic() > deadline - still_waiting = [] - for status in remaining: - if status.Succeeded(): - continue - if status.Failed(): - errors.append( - f"RDMA transfer failed: {status.Message()} " - f"(code={status.Code()})" - ) - continue - if timed_out: - errors.append( - f"RDMA transfer timed out after {timeout:.0f}s. " - f"Check NIC queue depth or reduce concurrency; " - f"adjust with kv_connector_extra_config.transfer_timeout." - ) - else: - still_waiting.append(status) - if errors: - break - remaining = still_waiting - if remaining: - time.sleep(0.001) - if errors: - raise TransferError( - f"{len(errors)}/{len(transfers_to_wait)} transfers failed:\n" - + "\n".join(errors) - ) - - def async_wait_reqid(self): - assert self.notify_port is not None, "Notify port cannot be None" - - if self.notify_thread is not None: - return - - def _async_wait(): - host = "*" - path = make_zmq_path("tcp", host, self.notify_port) - logger.info("Node starting to listen notify from path = %s", path) - - with zmq_ctx(zmq.ROUTER, path) as sock: - while True: - try: - identity, msg = sock.recv_multipart() - self._handle_message(msg) - except Exception as e: - logger.error("Error processing message: %s", e) - raise HandshakeError(f"Error processing message: {e}") from e - - self.notify_thread = threading.Thread( - target=_async_wait, daemon=True, name="moriio-notify-listener" - ) - self.notify_thread.start() - - def _handle_message(self, msg: bytes): - """Handles incoming messages from remote nodes.""" - # Handles incoming remote messages: - # Prefill Role: - # [write] mode: receives block information (allocation) - # [read] mode: receives block release messages from decode side - # Decode Role: - # [write] mode: receives KV cache write completion notifications - msg_str = repr(msg) - handled = False - try: - data = msgpack.loads(msg) - if isinstance(data, dict): - self._handle_structured_message(data) - return - except ( - msgpack.exceptions.ExtraData, - msgpack.exceptions.UnpackException, - ValueError, - ): - logger.debug("Failed to decode msgpack message, will try as string") - pass - - try: - msg_str = msg.decode("UTF-8") - if msg_str: - self._handle_completion_message(msg_str) - handled = True - except UnicodeDecodeError: - logger.warning("Received non-UTF8 message: %r", msg) - if not handled: - raise MoRIIOError(f"Unhandled message format: {msg_str}") - - def _handle_structured_message(self, data: dict): - message_type = data.get("type") - if message_type is None and "req_id" in data: - message_type = "remote_blocks" - - if message_type == "remote_blocks": - self._handle_remote_blocks_message(data) - elif message_type == "write_done": - self._handle_write_done_message(data) - elif message_type == "write_failed": - self._handle_write_failed_message(data) - elif message_type == "release": - self._handle_release_message(data) - else: - raise MoRIIOError(f"Unhandled structured message type: {message_type}") - - def _handle_remote_blocks_message(self, data: dict): - assert get_role() == ROLE.PRODUCER, "Only prefill can get block messages" - transfer_id = data["transfer_id"] - block_notify_list = data.get("block_notify_list", []) - decode_dp_rank = data.get("decode_rank", 0) - if not block_notify_list: - raise MoRIIOError( - "block_notify_list cannot be empty in remote allocate message" - ) - - with self.lock: - if self._is_transfer_terminal_locked(transfer_id): - logger.debug( - "Ignoring remote allocation for terminal transfer %s", - transfer_id, - ) - return - self.done_remote_allocate_req_dict[transfer_id] = RemoteAllocInfo( - block_ids=( - [block_notify_list] - if isinstance(block_notify_list[0], int) - else block_notify_list - ), - decode_dp_rank=decode_dp_rank, - ) - - def _handle_write_done_message(self, data: dict): - assert get_role() != ROLE.PRODUCER, ( - "Only decode can get WRITE completion messages" - ) - transfer_id = data["transfer_id"] - with self.lock: - self.done_write_cache_req_ids.append(transfer_id) - - def _handle_write_failed_message(self, data: dict): - assert get_role() != ROLE.PRODUCER, "Only decode can get WRITE failure messages" - transfer_id = data["transfer_id"] - with self.lock: - self.failed_write_cache_req_ids.append(transfer_id) - - def _handle_release_message(self, data: dict): - assert get_role() == ROLE.PRODUCER, ( - "Only prefill can get transfer release messages" - ) - transfer_id = data["transfer_id"] - consumer_tp_size = int(data.get("consumer_tp_size", 1)) - if consumer_tp_size <= 0: - raise MoRIIOError( - f"Invalid consumer_tp_size in release message: {consumer_tp_size}" - ) - with self.lock: - self.done_req_ids.append(MoRIIOTransferAck(transfer_id, consumer_tp_size)) - self.done_remote_allocate_req_dict.pop(transfer_id, None) - self._mark_transfer_terminal_locked(transfer_id) - - def fail_unallocated_write(self, transfer_id: TransferId) -> bool: - """Mark a WRITE transfer terminal and release its source blocks if - decode never sent its allocation. - - Returns False if the transfer is already terminal or the allocation - has arrived; the writer then drops or executes the task as usual. - """ - with self.lock: - if ( - self._is_transfer_terminal_locked(transfer_id) - or transfer_id in self.done_remote_allocate_req_dict - ): - return False - self._mark_transfer_terminal_locked(transfer_id) - self.done_req_ids.append(MoRIIOTransferAck(transfer_id)) - return True - - def _handle_completion_message(self, msg: str): - with self.lock: - if get_role() == ROLE.PRODUCER: - self.done_req_ids.append(MoRIIOTransferAck(msg)) - self.done_remote_allocate_req_dict.pop(msg, None) - self._mark_transfer_terminal_locked(msg) - else: - self.done_write_cache_req_ids.append(msg) - - def _is_transfer_terminal_locked(self, transfer_id: TransferId) -> bool: - return transfer_id in self._terminal_transfer_ids - - def _mark_transfer_terminal_locked(self, transfer_id: TransferId) -> None: - self._terminal_transfer_ids[transfer_id] = None - self._terminal_transfer_ids.move_to_end(transfer_id) - while len(self._terminal_transfer_ids) > _MAX_TERMINAL_TRANSFER_IDS: - self._terminal_transfer_ids.popitem(last=False) - - def send_notify( - self, - req_ids, - remote_ip, - remote_port, - message_type: str | None = None, - message_fields: dict[str, Any] | None = None, - ): - if not remote_ip or not remote_port: - logger.warning("Missing remote_ip or remote_port for notification") - return - - path = make_zmq_path("tcp", remote_ip, remote_port) - - if path not in self.paths: - ctx = zmq.Context.instance() - sock = make_zmq_socket( - ctx=ctx, path=path, socket_type=zmq.DEALER, bind=False - ) - self.paths[path] = sock - - req_list = req_ids if isinstance(req_ids, list) else [req_ids] - - sock = self.paths[path] - try: - for req_id in req_list: - if not isinstance(req_id, str): - logger.warning( - "Invalid req_id type: %s, expected str", type(req_id) - ) - continue - if message_type is None: - sock.send(req_id.encode("utf-8")) - else: - payload = {"type": message_type, "transfer_id": req_id} - if message_fields: - payload.update(message_fields) - sock.send(msgpack.dumps(payload)) - except Exception as e: - logger.error("Failed to send notification to %s: %s", path, e) - self.paths.pop(path, None) - raise - - def pop_finished_req_ids(self): - # Producer invocation: return every completion message since the last - # call. Do not dedupe: heterogeneous TP can produce multiple release - # ACKs for the same transfer_id and the caller must count each one. - with self.lock: - done_send = list(self.done_req_ids) - self.done_req_ids = [] - return done_send - - def pop_finished_write_req_ids(self): - # Call the consumer in write mode to get the collection after write completion - with self.lock: - done_write_cache = set(self.done_write_cache_req_ids) - self.done_write_cache_req_ids = [] - return done_write_cache - - def pop_failed_write_req_ids(self): - with self.lock: - failed_write_cache = set(self.failed_write_cache_req_ids) - self.failed_write_cache_req_ids = [] - return failed_write_cache - - def shutdown(self): - logger.debug("Closing MoRIIOWrapper and cleaning up ZMQ sockets") - for path, sock in self.paths.items(): - try: - sock.close(linger=0) - logger.debug("Closed ZMQ socket for path: %s", path) - except Exception as e: - logger.warning("Error closing ZMQ socket for path %s: %s", path, e) - self.paths.clear() diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_heartbeat.py b/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_heartbeat.py deleted file mode 100644 index 87eefe8f42..0000000000 --- a/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_heartbeat.py +++ /dev/null @@ -1,96 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Keep discovery heartbeats independent of the model worker's GIL.""" - -import json -import logging -import select -import subprocess -import sys -from typing import Any - -import msgpack -import zmq - -logger = logging.getLogger(__name__) - - -class MoRIIOHeartbeat: - def __init__( - self, - endpoint: str, - payload: dict[str, Any], - interval: float, - max_retries: int = 100, - ): - self._process = subprocess.Popen( - [sys.executable, __file__, endpoint, str(interval), str(max_retries)], - stdin=subprocess.PIPE, - text=True, - ) - assert self._process.stdin is not None - try: - self._process.stdin.write(json.dumps(payload) + "\n") - self._process.stdin.flush() - except BaseException: - self.shutdown() - raise - - def check_health(self) -> None: - returncode = self._process.poll() - if returncode is not None: - raise RuntimeError( - f"MoRIIO discovery heartbeat exited with code {returncode}" - ) - - def shutdown(self) -> None: - assert self._process.stdin is not None - try: - if not self._process.stdin.closed: - self._process.stdin.close() - except BrokenPipeError: - pass - finally: - try: - self._process.wait(timeout=5) - except subprocess.TimeoutExpired: - self._process.terminate() - try: - self._process.wait(timeout=1) - except subprocess.TimeoutExpired: - self._process.kill() - self._process.wait(timeout=1) - - -def main() -> None: - endpoint, interval_arg, max_retries_arg = sys.argv[1:] - initial = sys.stdin.readline() - if not initial: - return - payload = msgpack.dumps(json.loads(initial)) - interval = float(interval_arg) - max_retries = int(max_retries_arg) - failures = 0 - with zmq.Context() as context, context.socket(zmq.DEALER) as sock: - sock.setsockopt(zmq.LINGER, 0) - sock.setsockopt(zmq.SNDTIMEO, 1000) - sock.connect(endpoint) - while not select.select([sys.stdin], [], [], 0)[0]: - try: - sock.send(payload) - failures = 0 - except zmq.Again: - failures = 0 - logger.warning("MoRIIO discovery heartbeat send failed; retrying") - except zmq.ZMQError: - failures += 1 - logger.exception("MoRIIO discovery heartbeat send failed") - if failures >= max_retries: - raise - # EOF also stops the child when the worker exits without cleanup. - if select.select([sys.stdin], [], [], interval)[0]: - break - - -if __name__ == "__main__": - main() diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_layout.py b/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_layout.py deleted file mode 100644 index 8c55837dcd..0000000000 --- a/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_layout.py +++ /dev/null @@ -1,615 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from collections.abc import Callable, Mapping -from typing import NamedTuple, cast - -import torch - -from vllm.distributed.kv_transfer.kv_connector.v1.ssm_conv_transfer_utils import ( - MambaConvSplitInfo, -) -from vllm.v1.kv_cache_interface import ( - AttentionSpec, - KVCacheConfig, - KVCacheSpec, - MambaSpec, - MLAAttentionSpec, - SlidingWindowMLASpec, - UniformTypeKVCacheSpecs, -) - - -class LayerTransferGeometry(NamedTuple): - num_blocks: int - block_size: int - block_len: int - slot_size_bytes: int - block_stride: int - local_kv_stride: int | None - remote_kv_stride: int | None - transfers_per_block: int - regions_per_block: int - split_kv_regions: bool - - -class MambaTransferGeometry(NamedTuple): - """Geometry of a hybrid (mamba/KDA) layer's recurrent state. - - A KDA layer's packed page is exposed as ``conv_state`` and ``ssm_state`` - views. Each is slot-strided, so a slot's bytes live at - ``slot * slot_stride * element_size`` and span ``slot_bytes``. The two - tensors are registered as two separate MoRIIO regions per layer. - """ - - conv_slot_stride: int - ssm_slot_stride: int - ssm_slot_bytes: int - conv_element_size: int - ssm_element_size: int - conv_region_len: int - ssm_region_len: int - - -def get_mamba_transfer_geometry( - conv: torch.Tensor, - ssm: torch.Tensor, -) -> MambaTransferGeometry: - """Derive the slot-strided geometry of a KDA layer's conv + ssm state. - - ``conv``/``ssm`` are zero-copy views over the layer's packed page. Slot byte - offset is ``slot * slot_stride * element_size`` and each - slot spans ``subtensor[0].numel() * element_size`` bytes; the whole tensor - is registered as one region (``*_region_len`` bytes). - """ - if conv.ndim == 0 or ssm.ndim == 0 or conv.shape[0] == 0 or ssm.shape[0] == 0: - raise ValueError("KDA conv/ssm caches must contain at least one slot") - if conv.shape[0] != ssm.shape[0]: - raise ValueError( - f"KDA conv/ssm slot count mismatch: {conv.shape[0]} != {ssm.shape[0]}" - ) - conv_slot_bytes = conv[0].numel() * conv.element_size() - ssm_slot_bytes = ssm[0].numel() * ssm.element_size() - conv_available_bytes = ( - conv.untyped_storage().nbytes() - conv.storage_offset() * conv.element_size() - ) - ssm_available_bytes = ( - ssm.untyped_storage().nbytes() - ssm.storage_offset() * ssm.element_size() - ) - if conv.shape[0]: - conv_required_bytes = (conv.shape[0] - 1) * conv.stride( - 0 - ) * conv.element_size() + conv_slot_bytes - ssm_required_bytes = (ssm.shape[0] - 1) * ssm.stride( - 0 - ) * ssm.element_size() + ssm_slot_bytes - if conv_required_bytes > conv_available_bytes: - raise ValueError("KDA conv slots exceed the registered storage extent") - if ssm_required_bytes > ssm_available_bytes: - raise ValueError("KDA ssm slots exceed the registered storage extent") - - return MambaTransferGeometry( - conv_slot_stride=conv.stride(0), - ssm_slot_stride=ssm.stride(0), - ssm_slot_bytes=ssm_slot_bytes, - conv_element_size=conv.element_size(), - ssm_element_size=ssm.element_size(), - # Registered region must span the full slot-strided extent: stride(0) - # may exceed the per-slot element count, so numel() would under-size the - # region and truncate the highest-slot transfers. shape[0]*stride(0) - # covers every byte a ``slot * slot_stride * elem`` offset can address. - # Cap at the bytes remaining from each view's storage offset. A packed - # conv+ssm page places ssm at a non-zero intra-buffer offset, - # so do not register past the end of the shared backing allocation. - conv_region_len=min( - conv.shape[0] * conv.stride(0) * conv.element_size(), - conv_available_bytes, - ), - ssm_region_len=min( - ssm.shape[0] * ssm.stride(0) * ssm.element_size(), - ssm_available_bytes, - ), - ) - - -def build_layer_to_spec(kv_cache_config: KVCacheConfig) -> dict[str, KVCacheSpec]: - layer_to_spec: dict[str, KVCacheSpec] = {} - for group in kv_cache_config.kv_cache_groups: - group_spec = group.kv_cache_spec - if isinstance(group_spec, UniformTypeKVCacheSpecs): - layer_to_spec.update( - { - layer_name: group_spec.kv_cache_specs[layer_name] - for layer_name in group.layer_names - } - ) - else: - layer_to_spec.update( - {layer_name: group_spec for layer_name in group.layer_names} - ) - return layer_to_spec - - -def is_mla_cache_layer( - layer_to_spec: Mapping[str, KVCacheSpec], layer_name: str -) -> bool: - try: - spec = layer_to_spec[layer_name] - except KeyError as e: - raise ValueError(f"Missing KV cache spec for layer {layer_name}") from e - return isinstance(spec, (MLAAttentionSpec, SlidingWindowMLASpec)) - - -def _spec_dim_matches(value: int, expected: int | None) -> bool: - return expected is None or value == expected - - -def _kernel_layout_matches( - spec: KVCacheSpec, kernel_block_size: int, num_kv_heads: int, head_dim: int -) -> bool: - if kernel_block_size <= 0 or spec.block_size % kernel_block_size != 0: - return False - return _spec_dim_matches( - num_kv_heads, getattr(spec, "num_kv_heads", None) - ) and _spec_dim_matches(head_dim, getattr(spec, "head_size", None)) - - -def _select_kernel_block_layout( - layer_name: str, shape: torch.Size, spec: KVCacheSpec -) -> tuple[int, int, int]: - axis2_matches = _kernel_layout_matches(spec, shape[2], shape[3], shape[4]) - axis3_matches = _kernel_layout_matches(spec, shape[3], shape[2], shape[4]) - - if axis2_matches and axis3_matches and shape[2] != shape[3]: - raise ValueError( - f"Ambiguous MoRIIO kernel-block K/V cache shape for layer " - f"{layer_name}: {tuple(shape)}" - ) - if axis2_matches: - return shape[2], shape[3], shape[4] - if axis3_matches: - return shape[3], shape[2], shape[4] - - raise ValueError( - f"Unsupported MoRIIO K/V cache shape for layer {layer_name}: " - f"{tuple(shape)} does not contain block size {spec.block_size}" - ) - - -def get_layer_transfer_geometry( - layer_name: str, - kv_cache: torch.Tensor, - layer_to_spec: Mapping[str, KVCacheSpec], - remote_num_blocks: int | None = None, -) -> LayerTransferGeometry: - spec = layer_to_spec[layer_name] - shape = kv_cache.shape - stride = kv_cache.stride() - element_size = kv_cache.element_size() - is_mla_cache = is_mla_cache_layer(layer_to_spec, layer_name) - - if not is_mla_cache and len(shape) == 5 and shape[0] == 2: - _, num_blocks = shape[:2] - kernel_blocks_per_block = 1 - if shape[2] == spec.block_size: - block_size, num_kv_heads, head_dim = shape[2:] - elif shape[3] == spec.block_size: - num_kv_heads, block_size, head_dim = shape[2:] - else: - kernel_num_blocks = num_blocks - kernel_block_size, num_kv_heads, head_dim = _select_kernel_block_layout( - layer_name, shape, spec - ) - kernel_blocks_per_block = spec.block_size // kernel_block_size - if kernel_num_blocks % kernel_blocks_per_block != 0: - raise ValueError( - f"Unsupported MoRIIO K/V cache shape for layer {layer_name}: " - f"{tuple(shape)} has {kernel_num_blocks} kernel blocks, " - f"not divisible by {kernel_blocks_per_block}" - ) - num_blocks = kernel_num_blocks // kernel_blocks_per_block - block_size = spec.block_size - slot_size_bytes = num_kv_heads * head_dim * element_size - block_len = block_size * slot_size_bytes - return LayerTransferGeometry( - num_blocks=num_blocks, - block_size=block_size, - block_len=block_len, - slot_size_bytes=slot_size_bytes, - block_stride=stride[1] * kernel_blocks_per_block, - local_kv_stride=stride[0], - remote_kv_stride=( - stride[1] * kernel_blocks_per_block * (remote_num_blocks or num_blocks) - ), - transfers_per_block=2, - regions_per_block=1, - split_kv_regions=True, - ) - - if not is_mla_cache and len(shape) == 5 and shape[1] == 2: - num_blocks = shape[0] - if shape[2] == spec.block_size: - block_size, num_kv_heads, head_dim = shape[2:] - slot_size_bytes = num_kv_heads * head_dim * element_size - block_len = block_size * slot_size_bytes - return LayerTransferGeometry( - num_blocks=num_blocks, - block_size=block_size, - block_len=block_len, - slot_size_bytes=slot_size_bytes, - block_stride=stride[0], - local_kv_stride=stride[1], - remote_kv_stride=stride[1], - transfers_per_block=2, - regions_per_block=2, - split_kv_regions=False, - ) - elif shape[3] == spec.block_size: - num_kv_heads, block_size, head_dim = shape[2:] - else: - kernel_num_blocks = num_blocks - kernel_block_size, _, _ = _select_kernel_block_layout( - layer_name, shape, spec - ) - kernel_blocks_per_block = spec.block_size // kernel_block_size - if kernel_num_blocks % kernel_blocks_per_block != 0: - raise ValueError( - f"Unsupported MoRIIO K/V cache shape for layer {layer_name}: " - f"{tuple(shape)} has {kernel_num_blocks} kernel blocks, " - f"not divisible by {kernel_blocks_per_block}" - ) - num_blocks = kernel_num_blocks // kernel_blocks_per_block - block_size = spec.block_size - block_stride = stride[0] * kernel_blocks_per_block - block_len = block_stride * element_size - slot_size_bytes = block_len // block_size - return LayerTransferGeometry( - num_blocks=num_blocks, - block_size=block_size, - block_len=block_len, - slot_size_bytes=slot_size_bytes, - block_stride=block_stride, - local_kv_stride=None, - remote_kv_stride=None, - transfers_per_block=1, - regions_per_block=1, - split_kv_regions=False, - ) - slot_size_bytes = num_kv_heads * head_dim * element_size - block_len = block_size * slot_size_bytes - return LayerTransferGeometry( - num_blocks=num_blocks, - block_size=block_size, - block_len=block_len, - slot_size_bytes=slot_size_bytes, - block_stride=stride[0], - local_kv_stride=stride[1], - remote_kv_stride=stride[1], - transfers_per_block=2, - regions_per_block=2, - split_kv_regions=False, - ) - - if ( - isinstance(spec, AttentionSpec) - and len(shape) == 4 - and shape[1] == spec.num_heads - and shape[2] == spec.num_states - and shape[3] * element_size == spec.state_content_size_bytes - ): - # Standardized per-layer [B, H, N, C] view (MLA is just H == 1). - num_blocks, num_heads, num_states, content_dim = shape - slot_size_bytes = num_heads * content_dim * element_size - block_len = num_states * slot_size_bytes - return LayerTransferGeometry( - num_blocks=num_blocks, - block_size=spec.block_size, - block_len=block_len, - slot_size_bytes=slot_size_bytes, - block_stride=stride[0], - local_kv_stride=None, - remote_kv_stride=None, - transfers_per_block=1, - regions_per_block=1, - split_kv_regions=False, - ) - - cache_kind = "MLA" if is_mla_cache else "K/V" - raise ValueError( - f"Unsupported MoRIIO {cache_kind} cache shape for layer " - f"{layer_name}: {tuple(shape)}" - ) - - -def kda_conv_ssm( - kv_cache: torch.Tensor, - spec: MambaSpec, -) -> tuple[torch.Tensor, torch.Tensor]: - """Return zero-copy conv and SSM views over a packed two-state page.""" - from math import prod - - from vllm.utils.torch_utils import get_dtype_size - - if len(spec.shapes) != 2 or len(spec.dtypes) != 2: - raise ValueError( - "MoRIIO KDA transfer requires exactly two states (conv and SSM)" - ) - if kv_cache.element_size() != 1: - raise ValueError("MoRIIO KDA packed cache must use a byte-sized dtype") - if kv_cache.ndim != 4 or kv_cache.shape[1:3] != (1, 1) or kv_cache.stride(3) != 1: - raise ValueError( - "MoRIIO KDA packed cache must have shape [B, 1, 1, C] with " - "a contiguous byte dimension" - ) - # Mirror MambaBase.bind_kv_cache: blocks may be strided within a shared - # multi-layer allocation, while every individual byte page is contiguous. - pages = kv_cache.squeeze(dim=(1, 2)) - states: list[torch.Tensor] = [] - offset = 0 - for shp, dt in zip(spec.shapes, spec.dtypes, strict=True): - nbytes = prod(shp) * get_dtype_size(dt) - if offset + nbytes > pages.shape[1]: - raise ValueError( - "MoRIIO KDA state layout exceeds the packed cache page size" - ) - state = pages[:, offset : offset + nbytes].view(dt) - states.append(state.view(-1, *shp)) - offset += nbytes - return states[0], states[1] - - -def iter_layer_registration_regions( - layer_name: str, - kv_cache: torch.Tensor, - layer_to_spec: Mapping[str, KVCacheSpec], -) -> list[tuple[torch.Tensor, int]]: - spec = layer_to_spec[layer_name] - if isinstance(spec, MambaSpec): - # KDA layer: register the conv and ssm tensors as two whole-tensor - # regions (the conv 3-subprojection split is expressed as transfer - # offset triples, not extra registrations). - conv, ssm = kda_conv_ssm(kv_cache, spec) - geom = get_mamba_transfer_geometry(conv, ssm) - return [(conv, geom.conv_region_len), (ssm, geom.ssm_region_len)] - geometry = cast( - LayerTransferGeometry, - get_layer_transfer_geometry(layer_name, kv_cache, layer_to_spec), - ) - region_len = geometry.num_blocks * geometry.regions_per_block * geometry.block_len - if geometry.regions_per_block == 1: - # Padded/interleaved pages stride past block_len: register the strided - # span, ending at the last block's bytes so it stays in-allocation. - block_stride_bytes = geometry.block_stride * kv_cache.element_size() - region_len = max( - region_len, - (geometry.num_blocks - 1) * block_stride_bytes + geometry.block_len, - ) - if geometry.split_kv_regions: - return [(cache, region_len) for cache in kv_cache] - return [(kv_cache, region_len)] - - -def merge_contiguous_offsets( - offsets_local: list[int], - offsets_remote: list[int], - sizes: list[int], -) -> tuple[list[int], list[int], list[int]]: - if not offsets_local: - return [], [], [] - if not (len(offsets_local) == len(offsets_remote) == len(sizes)): - raise ValueError("Input list lengths mismatch") - - rows = sorted(zip(offsets_local, offsets_remote, sizes), key=lambda row: row[0]) - merged: list[list[int]] = [] - for local, remote, size in rows: - if ( - merged - and local == merged[-1][0] + merged[-1][2] - and remote == merged[-1][1] + merged[-1][2] - ): - merged[-1][2] += size - else: - merged.append([local, remote, size]) - - return ( - [row[0] for row in merged], - [row[1] for row in merged], - [row[2] for row in merged], - ) - - -def compute_block_transfer_offsets( - layer_name: str, - kv_cache: torch.Tensor, - layer_to_spec: Mapping[str, KVCacheSpec], - local_block_ids: list[int], - remote_block_ids: list[int], - remote_num_blocks: int, - merge_fn: Callable[ - [list[int], list[int], list[int]], tuple[list[int], list[int], list[int]] - ] = merge_contiguous_offsets, -) -> tuple[list[int], list[int], list[int]]: - # A shorter/empty local list transfers only what decode allocated (READ-mode - # partial or full-prefix-hit); a longer local list is a genuine bug. - if len(local_block_ids) > len(remote_block_ids): - raise ValueError( - "local_block_ids longer than remote_block_ids: " - f"{len(local_block_ids)} > {len(remote_block_ids)}" - ) - geometry = cast( - LayerTransferGeometry, - get_layer_transfer_geometry( - layer_name, kv_cache, layer_to_spec, remote_num_blocks - ), - ) - invalid_local = next( - ( - block_id - for block_id in local_block_ids - if not 0 <= block_id < geometry.num_blocks - ), - None, - ) - if invalid_local is not None: - raise ValueError( - f"local block id {invalid_local} is outside [0, {geometry.num_blocks})" - ) - invalid_remote = next( - ( - block_id - for block_id in remote_block_ids - if not 0 <= block_id < remote_num_blocks - ), - None, - ) - if invalid_remote is not None: - raise ValueError( - f"remote block id {invalid_remote} is outside [0, {remote_num_blocks})" - ) - element_size = kv_cache.element_size() - transfer_size_byte = geometry.block_len - per_block = geometry.transfers_per_block - total = len(local_block_ids) * per_block - offset_local = [0] * total - offset_remote = [0] * total - sizes = [transfer_size_byte] * total - - w = 0 - for lb, rb in zip(local_block_ids, remote_block_ids): - offset_local[w] = element_size * (lb * geometry.block_stride) - offset_remote[w] = element_size * (rb * geometry.block_stride) - w += 1 - if per_block == 2: - assert geometry.local_kv_stride is not None - assert geometry.remote_kv_stride is not None - offset_local[w] = element_size * ( - geometry.local_kv_stride + lb * geometry.block_stride - ) - offset_remote[w] = element_size * ( - geometry.remote_kv_stride + rb * geometry.block_stride - ) - w += 1 - - return merge_fn(offset_local, offset_remote, sizes) - - -class MambaOffsetTemplate(NamedTuple): - """Slot-independent conv+ssm offset decomposition for a KDA layer. - - Homogeneous-TP geometry (conv/ssm slot strides, conv sub-projection - offsets, ssm per-slot size) is identical across requests and across the - homogeneous GDN layers, so it is computed once and reused; only the - per-request slot bases vary (see ``apply_mamba_offset_template``). - """ - - num_slots: int - conv_slot_stride_bytes: int - ssm_slot_stride_bytes: int - ssm_read_bytes: int - conv_subprojs: tuple[tuple[int, int, int], ...] - - -def build_mamba_offset_template( - conv: torch.Tensor, - ssm: torch.Tensor, - split_info: MambaConvSplitInfo, - tp_ratio: int, -) -> MambaOffsetTemplate: - """Precompute the slot-independent conv+ssm offset decomposition. - - Only homogeneous TP (``tp_ratio == 1``) is supported; heterogeneous TP is - gated with ``NotImplementedError`` (see the design doc). The returned - template is a pure function of the layer geometry and ``split_info``, so it - can be cached and reused across requests without recomputation. - """ - if tp_ratio != 1: - # Heterogeneous-TP recurrent-state transfer needs the remote page's - # slot stride (which differs from the local page) and, for - # P_TP > D_TP, a multi-rank gather of conv/ssm slices. Both are - # follow-ups. The mamba path bypasses the attention KV-head validator, - # so fail loudly here instead of silently transferring wrong bytes. - raise NotImplementedError( - "heterogeneous-TP mamba/KDA transfer is not supported yet " - f"(tp_ratio={tp_ratio}); use equal prefill/decode TP for " - "hybrid models" - ) - geom = get_mamba_transfer_geometry(conv, ssm) - conv_local_offsets = split_info.local_conv_offsets - conv_remote_offsets = split_info.remote_conv_offsets(0, 1) - ssm_read_bytes = geom.ssm_slot_bytes - conv_subprojs = tuple( - (loff, roff, rsz) - for (loff, _lsz), (roff, rsz) in zip(conv_local_offsets, conv_remote_offsets) - ) - return MambaOffsetTemplate( - num_slots=conv.shape[0], - conv_slot_stride_bytes=geom.conv_slot_stride * geom.conv_element_size, - ssm_slot_stride_bytes=geom.ssm_slot_stride * geom.ssm_element_size, - ssm_read_bytes=ssm_read_bytes, - conv_subprojs=conv_subprojs, - ) - - -def apply_mamba_offset_template( - template: MambaOffsetTemplate, - local_slots: list[int], - remote_slots: list[int], -) -> tuple[list[int], list[int], list[int]]: - """Apply per-request slot bases to a cached ``MambaOffsetTemplate``. - - Conv sub-projection entries (all slots) come first, followed by one ssm - entry per slot. - """ - if len(local_slots) != len(remote_slots): - raise ValueError( - "local and remote mamba slot counts must match: " - f"{len(local_slots)} != {len(remote_slots)}" - ) - invalid_local = next( - (slot for slot in local_slots if not 0 <= slot < template.num_slots), None - ) - if invalid_local is not None: - raise ValueError( - f"local mamba slot {invalid_local} is outside [0, {template.num_slots})" - ) - invalid_remote = next((slot for slot in remote_slots if slot < 0), None) - if invalid_remote is not None: - raise ValueError(f"remote mamba slot {invalid_remote} must be non-negative") - # The peer's KDA slot count is not part of MoRIIOAgentMetadata yet. Do not - # compare remote slot IDs against the local pool size: two otherwise - # compatible engines may reserve different numbers of slots. - local_offs: list[int] = [] - remote_offs: list[int] = [] - sizes: list[int] = [] - - # Conv sub-projections (region 0), all slots first. - conv_slot_stride_bytes = template.conv_slot_stride_bytes - for ls, rs in zip(local_slots, remote_slots): - lbase = ls * conv_slot_stride_bytes - rbase = rs * conv_slot_stride_bytes - for loff, roff, rsz in template.conv_subprojs: - local_offs.append(lbase + loff) - remote_offs.append(rbase + roff) - sizes.append(rsz) - - # SSM temporal state (region 1), one entry per slot. - ssm_slot_stride_bytes = template.ssm_slot_stride_bytes - ssm_read_bytes = template.ssm_read_bytes - for ls, rs in zip(local_slots, remote_slots): - local_offs.append(ls * ssm_slot_stride_bytes) - remote_offs.append(rs * ssm_slot_stride_bytes) - sizes.append(ssm_read_bytes) - - return local_offs, remote_offs, sizes - - -def compute_mamba_conv_split_count( - local_slots: list[int], - split_info: MambaConvSplitInfo, -) -> int: - """Number of leading conv entries in a KDA layer transfer plan. - - Entries ``[:count]`` are conv sub-projections (conv region/session); - entries ``[count:]`` are the per-slot ssm state (ssm region/session). - """ - return len(local_slots) * len(split_info.local_conv_offsets) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/__init__.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/__init__.py deleted file mode 100644 index fd5996f64b..0000000000 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/__init__.py +++ /dev/null @@ -1,61 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""NIXL KV-cache transfer connector (disaggregated prefill / decode).""" - -from vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_scheduler import ( - NixlBaseConnectorScheduler, -) -from vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker import ( - NixlBaseConnectorWorker, -) -from vllm.distributed.kv_transfer.kv_connector.v1.nixl.connector import ( - NixlBaseConnector, - NixlConnector, - NixlPullConnector, - NixlPushConnector, -) -from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import ( - NixlAgentMetadata, - NixlConnectorMetadata, - NixlHandshakePayload, -) -from vllm.distributed.kv_transfer.kv_connector.v1.nixl.pull_scheduler import ( - NixlPullConnectorScheduler, -) -from vllm.distributed.kv_transfer.kv_connector.v1.nixl.pull_worker import ( - NixlPullConnectorWorker, -) -from vllm.distributed.kv_transfer.kv_connector.v1.nixl.push_scheduler import ( - NixlPushConnectorScheduler, -) -from vllm.distributed.kv_transfer.kv_connector.v1.nixl.push_worker import ( - NixlPushConnectorWorker, -) -from vllm.distributed.kv_transfer.kv_connector.v1.nixl.scheduler import ( - NixlConnectorScheduler, -) -from vllm.distributed.kv_transfer.kv_connector.v1.nixl.stats import ( - NixlKVConnectorStats, -) -from vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker import ( - NixlConnectorWorker, -) - -__all__ = [ - "NixlAgentMetadata", - "NixlBaseConnector", - "NixlBaseConnectorScheduler", - "NixlBaseConnectorWorker", - "NixlConnector", - "NixlConnectorMetadata", - "NixlConnectorScheduler", - "NixlConnectorWorker", - "NixlHandshakePayload", - "NixlKVConnectorStats", - "NixlPullConnector", - "NixlPullConnectorScheduler", - "NixlPullConnectorWorker", - "NixlPushConnector", - "NixlPushConnectorScheduler", - "NixlPushConnectorWorker", -] diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_scheduler.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_scheduler.py deleted file mode 100644 index 55ee98d465..0000000000 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_scheduler.py +++ /dev/null @@ -1,549 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Base scheduler-side logic for the NIXL connector.""" - -import threading -import time -from typing import TYPE_CHECKING, Any - -import msgspec -import zmq - -from vllm import envs -from vllm.distributed.kv_transfer.kv_connector.utils import ( - BlockIds, - EngineId, - clip_ssm_state_blocks, - yield_req_data, -) -from vllm.distributed.kv_transfer.kv_connector.v1.base import ( - KVConnectorHandshakeMetadata, - KVConnectorMetadata, -) -from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import ( - GET_META_MSG, - HeartbeatInfo, - NixlConnectorMetadata, - NixlHandshakePayload, - ReqId, -) -from vllm.distributed.kv_transfer.kv_connector.v1.nixl.utils import zmq_ctx -from vllm.logger import init_logger -from vllm.platforms import current_platform -from vllm.utils.math_utils import cdiv -from vllm.utils.network_utils import make_zmq_path -from vllm.v1.core.sched.output import SchedulerOutput -from vllm.v1.kv_cache_interface import ( - FullAttentionSpec, - MambaSpec, - SlidingWindowSpec, -) - -if TYPE_CHECKING: - from vllm.config import VllmConfig - from vllm.v1.core.kv_cache_manager import KVCacheBlocks - from vllm.v1.kv_cache_interface import KVCacheConfig - from vllm.v1.outputs import KVConnectorOutput - from vllm.v1.request import Request - -logger = init_logger(__name__) - - -class NixlBaseConnectorScheduler: - """Base implementation of Scheduler side methods shared by pull and push.""" - - # Emitted in kv_transfer_params so an external router can distinguish a - # pull (READ) producer from a push (WRITE) one. Overridden by the push - # scheduler. - _TRANSFER_MODE: str = "pull" - - def __init__( - self, - vllm_config: "VllmConfig", - engine_id: str, - kv_cache_config: "KVCacheConfig", - ): - self.vllm_config = vllm_config - parallel_config = vllm_config.parallel_config - # TP1 PCP+DCP exposes its DCP shards as transfer ranks. - self.transfer_tp_size = max( - parallel_config.tensor_parallel_size, - parallel_config.decode_context_parallel_size, - ) - self.block_size = vllm_config.cache_config.block_size - self.engine_id: EngineId = engine_id - self.kv_cache_config = kv_cache_config - self.side_channel_host = envs.VLLM_NIXL_SIDE_CHANNEL_HOST - self.side_channel_port = ( - envs.VLLM_NIXL_SIDE_CHANNEL_PORT - + vllm_config.parallel_config.data_parallel_index - ) - assert vllm_config.kv_transfer_config is not None - self._kv_lease_duration: int = ( - vllm_config.kv_transfer_config.get_from_extra_config( - "kv_lease_duration", 30 - ) - ) - # NOTE (NickLucche): For now we use a hardcoded value for a simpler interface. - self._heartbeat_interval = self._kv_lease_duration // 6 - if current_platform.device_type == "cpu": - self.use_host_buffer = False - else: - self.use_host_buffer = ( - vllm_config.kv_transfer_config.kv_buffer_device == "cpu" - ) - self._is_hma_required = ( - not vllm_config.scheduler_config.disable_hybrid_kv_cache_manager - # Also handle unlikely SW-only model case instead of checking num_groups>1. - and any( - not isinstance(g.kv_cache_spec, FullAttentionSpec) - for g in kv_cache_config.transfer_groups - ) - ) - self._has_mamba = kv_cache_config.has_mamba_layers - self._bounded_replay = any( - g.kv_cache_spec.prefix_replay_tokens > 0 - for g in kv_cache_config.kv_cache_groups - ) - - logger.info("Initializing NIXL Scheduler %s", engine_id) - if vllm_config.scheduler_config.disable_hybrid_kv_cache_manager: - logger.info("Hybrid Memory Allocator is enabled with NIXL") - - # Background thread for handling new handshake requests. - self._nixl_handshake_listener_t: threading.Thread | None = None - self._stop_event = threading.Event() - - # Requests that need to start recv/send. - # New requests are added by update_state_after_alloc in - # the scheduler. Used to make metadata passed to Worker. - self._reqs_need_recv: dict[ - ReqId, tuple[Request, BlockIds, tuple[int, ...], bool] - ] = {} - self._reqs_need_save: dict[ReqId, Request] = {} - # Reqs to send and their expiration time - self._reqs_need_send: dict[ReqId, float] = {} - self._reqs_in_batch: set[ReqId] = set() - # Reqs to remove from processed set because they're not to send after - # remote prefill or aborted. - self._reqs_not_processed: set[ReqId] = set() - - # Heartbeat tracking: requests needing periodic lease-renewal heartbeats to - # remote P-side, stored as ready-to-send HeartbeatInfo grouped by remote engine - self._heartbeat_by_engine: dict[EngineId, HeartbeatInfo] = {} - # Reverse lookup: local req_id -> (engine_id, remote_req_id) for O(1) removal - self._heartbeat_req_engine: dict[ReqId, tuple[EngineId, ReqId]] = {} - self._last_heartbeat_time: float = 0.0 - - # Gather Sliding Window sizes for each kv cache group (if any) in number of - # blocks per KV cache group. This is used to clip the local attention window. - sw_sizes_tokens: list[tuple[int, int]] = [ - (g.kv_cache_spec.sliding_window, g.kv_cache_spec.block_size) - if isinstance(g.kv_cache_spec, SlidingWindowSpec) - else (0, self.block_size) - for g in kv_cache_config.transfer_groups - ] - # cdiv(n_tokens, block_size) gives blocks/window; add 1 to conservatively - # account for boundary overlap eg window isn't fully aligned with blocks. - self.blocks_per_sw = [ - cdiv(n_tokens, block_size) + 1 if n_tokens else 0 - for n_tokens, block_size in sw_sizes_tokens - ] - - # Trailing scratch slots that mamba managers co-allocate per request - # for speculative decoding; None for non-SSM groups. - self._ssm_spec_blocks = [ - g.kv_cache_spec.num_speculative_blocks - if isinstance(g.kv_cache_spec, MambaSpec) - else None - for g in kv_cache_config.transfer_groups - ] - - # Threshold to decide whether to compute kv cache locally - # or pull from a remote node: minimum number of remote - # tokens to amortize the xfer latencies - self.kv_recompute_threshold: int = int( - vllm_config.kv_transfer_config.get_from_extra_config( - "kv_recompute_threshold", 64 - ) - ) - - # Bi-directional KV transfer feature supports KV block - # transfers from D node to P node - self.is_bidirectional_kv_xfer_enabled = ( - vllm_config.kv_transfer_config.get_from_extra_config( - "bidirectional_kv_xfer", False - ) - ) - self.decoder_kv_blocks_ttl = ( - vllm_config.kv_transfer_config.get_from_extra_config( - "decoder_kv_blocks_ttl", 480 - ) - ) - - if self.is_bidirectional_kv_xfer_enabled and self.kv_recompute_threshold > 0: - logger.info( - "Bidirectional KV transfer is enabled and the kv " - "recompute threshold is set to %d tokens." - "KV blocks on D are released after a TTL of %d seconds.", - self.kv_recompute_threshold, - self.decoder_kv_blocks_ttl, - ) - - def shutdown(self): - self._stop_event.set() - if self._nixl_handshake_listener_t is not None: - self._nixl_handshake_listener_t.join() - self._nixl_handshake_listener_t = None - - def on_new_request(self, request: "Request") -> None: - """Track a request that may need heartbeats.""" - params = request.kv_transfer_params - if params is not None and params.get("do_remote_decode"): - self._truncate_request_for_prefill(request) - - # NOTE (NickLucche) This excludes request meant for P, ie heartbeats are - # effectively disabled for Bidirectional KV transfer. - if params is None or not params.get("do_remote_prefill"): - return - # Only track if all required remote fields are present. - remote_engine_id = params.get("remote_engine_id") - remote_request_id = params.get("remote_request_id") - host = params.get("remote_host") - port = params.get("remote_port") - tp_size = params.get("tp_size") - dcp_size = params.get("dcp_size", 1) - pp_size = params.get("pp_size", 1) - if ( - remote_engine_id is None - or remote_request_id is None - or host is None - or port is None - or tp_size is None - ): - return - if remote_engine_id not in self._heartbeat_by_engine: - self._heartbeat_by_engine[remote_engine_id] = HeartbeatInfo( - req_ids=set(), - host=host, - port=port, - tp_size=tp_size, - dcp_size=dcp_size, - pp_size=pp_size, - ) - self._heartbeat_by_engine[remote_engine_id].req_ids.add(remote_request_id) - self._heartbeat_req_engine[request.request_id] = ( - remote_engine_id, - remote_request_id, - ) - - def _stop_heartbeat(self, req_id: ReqId) -> None: - """Remove *req_id* from heartbeat tracking (if tracked).""" - if key := self._heartbeat_req_engine.pop(req_id, None): - engine_id, remote_id = key - if info := self._heartbeat_by_engine.get(engine_id): - info.req_ids.discard(remote_id) - if not info.req_ids: - # Clean up empty engines so we don't leak a key when remote dies. - del self._heartbeat_by_engine[engine_id] - - def get_exchange_clipped_blocks( - self, block_ids: BlockIds, clip_ssm: bool = True - ) -> BlockIds: - """Clip a request's block lists down to the transferable blocks. - - Sliding-window groups keep only the in-window tail: the KV cache - manager allocates blocks for the entire sequence length and cleans up - out-of-window blocks only prior to the `request_finished_all_groups` - hook. - - SSM groups keep only their state-bearing slot: the trailing - speculative scratch slots and everything before the running state - (null placeholders and the previous step's superseded state) go. - - Use this at every block-id exchange point. Pass ``clip_ssm=False`` - for per-step partial lists (host-buffer save), where the SSM strip - does not apply. - """ - if len(block_ids) == 0: - # No blocks to clip, e.g. a full prefix cache hit. - return block_ids - block_ids = self.kv_cache_config.select_transfer_block_ids(block_ids) - if not self._is_hma_required: - return block_ids - # NOTE (NickLucche) This logic is currently handled at the connector level - # because offloading connectors might want to receive the whole sequence even - # for SWA groups. We will abstract this logic once the interface is more stable - assert len(block_ids) == len(self.blocks_per_sw), ( - "Number of KV cache groups must match" - ) - clipped = [] - for i, blocks in enumerate(block_ids): - if n_sw := self.blocks_per_sw[i]: - blocks = blocks[-n_sw:] - elif ( - clip_ssm - and blocks - and (n_spec_blocks := self._ssm_spec_blocks[i]) is not None - ): - blocks = clip_ssm_state_blocks(blocks, n_spec_blocks) - clipped.append(blocks) - return tuple(clipped) - - def set_xfer_handshake_metadata( - self, metadata: dict[tuple[int, int], KVConnectorHandshakeMetadata] - ) -> None: - """Set the KV connector handshake metadata for this connector. - - Args: - metadata (dict): the handshake metadata to set. - - """ - encoded_data: dict[tuple[int, int], bytes] = {} - encoder = msgspec.msgpack.Encoder() - for (pp_rank, tp_rank), rank_metadata in metadata.items(): - if not isinstance(rank_metadata, NixlHandshakePayload): - raise ValueError( - "NixlConnectorScheduler expects NixlHandshakePayload for " - "handshake metadata." - ) - encoded_data[(pp_rank, tp_rank)] = encoder.encode(rank_metadata) - logger.debug( - "PP rank %d, TP rank %d: encoded NixlHandshakePayload size: %s bytes", - pp_rank, - tp_rank, - str(len(encoded_data[(pp_rank, tp_rank)])), - ) - - # Only start the listener when we have metadata to serve. - if self._nixl_handshake_listener_t is None: - ready_event = threading.Event() - self._nixl_handshake_listener_t = threading.Thread( - target=self._nixl_handshake_listener, - args=( - encoded_data, - ready_event, - self._stop_event, - self.side_channel_host, - self.side_channel_port, - ), - daemon=True, - name="nixl_handshake_listener", - ) - self._nixl_handshake_listener_t.start() - ready_event.wait() # Wait for listener ZMQ socket to be ready. - - @staticmethod - def _nixl_handshake_listener( - encoded_data: dict[tuple[int, int], Any], - ready_event: threading.Event, - stop_event: threading.Event, - host: str, - port: int, - ): - """Background thread for getting new NIXL handshakes.""" - # NOTE(rob): this is a simple implementation. We will move - # to a better approach via HTTP endpoint soon. - - # Listen for new requests for metadata. - path = make_zmq_path("tcp", host, port) - logger.debug("Starting listening on path: %s", path) - with zmq_ctx(zmq.ROUTER, path) as sock: - sock.setsockopt(zmq.RCVTIMEO, 1000) - ready_event.set() - while True: - try: - identity, _, msg = sock.recv_multipart() - except zmq.Again: - if stop_event.is_set(): - break - continue - # Decode (GET_META_MSG, pp_rank, tp_rank). - msg, target_pp_rank, target_tp_rank = msgspec.msgpack.decode(msg) - logger.debug( - "Received message for pp rank %s, tp rank %s", - target_pp_rank, - target_tp_rank, - ) - if msg != GET_META_MSG: - logger.warning("Connection listener got unexpected message %s", msg) - # Echo our perf_counter so P can estimate the clock offset. - # perf_counter is only comparable within a process, so this - # listener must run in the same process that stamps the block - # expiry deadline (`_reqs_need_send`). - ts = msgspec.msgpack.encode(time.perf_counter()) - sock.send_multipart( - (identity, b"", encoded_data[(target_pp_rank, target_tp_rank)], ts) - ) - - def _prefill_backoff(self) -> int: - """Trailing prompt tokens the prefiller must not compute; the decoder - recomputes them locally. - - Mamba needs h(N-1) so the decoder can derive h(N) itself. DSV41's sliding - window under bounded replay is not replayed after a P/D load, so it must - be laid out for the token the decoder recomputes. Multi-module - MTP needs to keep its whole lookahead window off of the prefiller, which - would otherwise embed the unverified drafts in the MTP layer's KV cache. The - decoder would never rebuild them, because the update is sized by the rejection - count, which is zero for the first decode. - """ - return max( - 1 if self._has_mamba or self._bounded_replay else 0, - self.vllm_config.num_prefill_lookahead_tokens - 1, - ) - - def _get_remote_prefill_token_count(self, num_prompt_tokens: int) -> int: - """D-side only. The number of prompt tokens to load from the prefiller. - Stops short of the trailing ``_prefill_backoff()`` tokens that the decoder - will recompute locally.""" - backoff = self._prefill_backoff() - if backoff and num_prompt_tokens > backoff: - return num_prompt_tokens - backoff - return num_prompt_tokens - - def _truncate_request_for_prefill(self, request: "Request") -> None: - """P-side only: drop the trailing ``_prefill_backoff()`` prompt tokens - so the prefiller stops short of what the decoder recomputes locally. - For Mamba that is the single token needed to yield h(N-1); for - multi-module MTP it is the drafter's whole lookahead window. - - Guarded by ``_p_side_truncated`` to avoid repeated truncation if the - request is preempted and rescheduled.""" - backoff = self._prefill_backoff() - params = request.kv_transfer_params - if ( - backoff - and params is not None - # Guard against repeated truncation after preemption/reschedule. - and not params.get("_p_side_truncated") - and request.num_prompt_tokens > backoff - # A request that skips reading the prefix cache (prompt logprobs) - # loads nothing on the decoder, which recomputes its whole prompt. - and not request.get_skip_reading_prefix_cache() - ): - if request.prompt_token_ids is not None: - del request.prompt_token_ids[-backoff:] - elif request.prompt_embeds is not None: - request.prompt_embeds = request.prompt_embeds[:-backoff] - else: - return - - del request._all_token_ids[-backoff:] - request.num_prompt_tokens -= backoff - request.max_tokens = 1 - params["_p_side_truncated"] = True - - def _build_save_meta( - self, - meta: NixlConnectorMetadata, - scheduler_output: SchedulerOutput, - ) -> None: - # only called when use_host_buffer is True to build the save metadata - - # NOTE: For the prefill side, there might be a chance that an early added - # request is a chunked prefill, so we need to check if new blocks are added - for req_id, new_block_id_groups, _ in yield_req_data(scheduler_output): - req_to_save = self._reqs_need_save.get(req_id) - if req_to_save is None or new_block_id_groups is None: - continue - req = req_to_save - - assert req.kv_transfer_params is not None - clipped_block_id_groups = self.get_exchange_clipped_blocks( - new_block_id_groups, clip_ssm=False - ) - meta.add_new_req_to_save( - request_id=req_id, - local_block_ids=clipped_block_id_groups, - kv_transfer_params=req.kv_transfer_params, - ) - assert scheduler_output.num_scheduled_tokens is not None - num_scheduled_tokens = scheduler_output.num_scheduled_tokens[req_id] - is_partial = ( - req.num_computed_tokens + num_scheduled_tokens - ) < req.num_prompt_tokens - if not is_partial: - # For non-partial prefills, once new req_meta is scheduled, it - # can be removed from _reqs_need_save. - # For partial prefill case, we will retain the request in - # _reqs_need_save until all blocks are scheduled with req_meta. - # Therefore, only pop if `not is_partial`. - self._reqs_need_save.pop(req_id) - - def build_connector_meta( - self, - scheduler_output: SchedulerOutput, - ) -> KVConnectorMetadata: - meta = NixlConnectorMetadata() - - # Loop through scheduled reqs and convert to ReqMeta. - for req_id, ( - req, - block_ids, - cached, - awaiting_kvs, - ) in self._reqs_need_recv.items(): - assert req.kv_transfer_params is not None - meta.add_new_req_to_recv( - request_id=req_id, - local_block_ids=block_ids, - kv_transfer_params=req.kv_transfer_params, - local_num_computed_blocks=cached, - awaiting_kvs=awaiting_kvs, - ) - - if self.use_host_buffer: - self._build_save_meta(meta, scheduler_output) - - meta.reqs_to_send = self._reqs_need_send - # Clock reference for reqs_to_send: deadlines above are in this - # process's perf_counter domain; workers (possibly on other nodes, - # where perf_counter has a different epoch) rebase against this. - meta.scheduler_clock = time.perf_counter() - meta.reqs_in_batch = self._reqs_in_batch - meta.reqs_not_processed = self._reqs_not_processed - - # Package heartbeats, throttled by heartbeat_interval. - if self._heartbeat_by_engine: - now = time.perf_counter() - if now - self._last_heartbeat_time >= self._heartbeat_interval: - self._last_heartbeat_time = now - meta.heartbeat_by_engine = self._heartbeat_by_engine - - # Clear the list once workers start the transfers - self._reqs_need_recv.clear() - self._reqs_in_batch = set() - self._reqs_not_processed = set() - self._reqs_need_send = {} - - return meta - - def update_connector_output(self, connector_output: "KVConnectorOutput") -> None: - """Stop heartbeating for requests whose KV transfer completed.""" - for req_id in connector_output.finished_recving or (): - self._stop_heartbeat(req_id) - - def has_pending_push_work(self) -> bool: - return False - - ############################################################ - # Abstract methods that subclasses must implement - ############################################################ - - def get_num_new_matched_tokens( - self, request: "Request", num_computed_tokens: int - ) -> tuple[int, bool]: - raise NotImplementedError - - def update_state_after_alloc( - self, request: "Request", blocks: "KVCacheBlocks", num_external_tokens: int - ): - raise NotImplementedError - - def request_finished( - self, - request: "Request", - block_ids: BlockIds, - ) -> tuple[bool, dict[str, Any] | None]: - raise NotImplementedError diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py deleted file mode 100644 index 4e86000bb1..0000000000 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py +++ /dev/null @@ -1,3922 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Base worker-side logic for the NIXL connector.""" - -import contextlib -import itertools -import logging -import math -import os -import queue -import threading -import time -import uuid -from collections import defaultdict -from collections.abc import Iterator -from concurrent.futures import Future, ThreadPoolExecutor -from typing import TYPE_CHECKING, Any, cast - -import msgspec -import numpy as np -import regex as re -import torch -import zmq - -from vllm.distributed.kv_transfer.kv_connector.utils import ( - BlockIds, - EngineId, - EngineTransferInfo, - TransferTopology, - get_current_attn_backends, - get_current_attn_backends_and_specs, - kv_postprocess_blksize_and_layout_on_receive, - kv_postprocess_blksize_on_receive, -) -from vllm.distributed.kv_transfer.kv_connector.v1.base import ( - CopyBlocksOp, - KVConnectorTransferResults, -) -from vllm.distributed.kv_transfer.kv_connector.v1.metrics import KVConnectorStats -from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import ( - GET_META_MSG, - NixlAgentMetadata, - NixlConnectorMetadata, - NixlHandshakePayload, - ReqId, - ReqMeta, - TransferHandle, - compute_nixl_compatibility_hash, -) -from vllm.distributed.kv_transfer.kv_connector.v1.nixl.stats import ( - NixlKVConnectorStats, -) -from vllm.distributed.kv_transfer.kv_connector.v1.nixl.tp_mapping import ( - TPMapping, - _is_attention_spec, - _is_ssm_spec, - compute_tp_mapping, -) -from vllm.distributed.kv_transfer.kv_connector.v1.nixl.utils import ( - _NIXL_SUPPORTED_DEVICE, - get_representative_spec_type, - zmq_ctx, -) -from vllm.distributed.kv_transfer.kv_connector.v1.ssm_conv_transfer_utils import ( - MambaConvSplitInfo, - derive_mamba_conv_split, -) -from vllm.distributed.nixl_utils import ( - NixlWrapper, - check_nixl_version, - nixl_agent_config, -) -from vllm.distributed.parallel_state import ( - get_pcp_group, - get_tensor_model_parallel_rank, - get_tensor_model_parallel_world_size, -) -from vllm.logger import init_logger -from vllm.platforms import current_platform -from vllm.utils.gpu_sync_debug import gpu_sync_allowed -from vllm.utils.math_utils import cdiv -from vllm.utils.network_utils import make_zmq_path -from vllm.utils.torch_utils import async_tensor_h2d -from vllm.v1.kv_cache_interface import ( - AttentionSpec, - CircularBufferSpec, - FullAttentionSpec, - KVCacheLayout, - KVCacheSpec, - MambaSpec, - MLAAttentionSpec, - SlidingWindowMLASpec, - UniformTypeKVCacheSpecs, - iter_layer_specs, -) -from vllm.v1.worker.utils import select_common_block_size - -if TYPE_CHECKING: - from vllm.config import VllmConfig - from vllm.v1.kv_cache_interface import KVCacheConfig - -logger = init_logger(__name__) - -_SHARED_REGION_GROUP_ID = -1 - -# Re-handshake delay bounds for a pull peer whose local metadata keeps getting -# invalidated, eg due to a persistent RDMA fault. -_INVALID_PEER_MIN_BACKOFF_S = 1.0 -_INVALID_PEER_MAX_BACKOFF_S = 60.0 - - -def _region_sort_key(layer_name: str) -> tuple[tuple[int, int | str], ...]: - """Sort transfer regions in model-layer order, then by cache name.""" - return tuple( - (0, int(part)) if part.isdigit() else (1, part) - for part in re.split(r"(\d+)", layer_name) - ) - - -def _share_storage_and_block_stride(caches: list[torch.Tensor]) -> bool: - """Return whether all views share storage and a block stride.""" - block_strides = {cache.stride(0) * cache.element_size() for cache in caches} - storage_ptrs = {cache.untyped_storage().data_ptr() for cache in caches} - return len(block_strides) == len(storage_ptrs) == 1 - - -class NixlBaseConnectorWorker: - """Base implementation of Worker side methods shared by pull and push.""" - - # Transfer mode included in the NIXL compatibility hash so that a push - # (WRITE) connector and a pull (READ) connector never handshake together. - # Overridden by NixlPushConnectorWorker. - _TRANSFER_MODE: str = "pull" - - # Layer-name routing is supported only by NixlPushConnector for HMA or - # packed KV caches under PP. - _supports_pp_hma = False - - def _compute_desc_ids( - self, - block_ids: BlockIds, - dst_num_blocks: int, - block_size_ratio: float | None, - physical_blocks_per_logical: int, - region_num_blocks: list[int] | None = None, - region_group_ids: list[int] | None = None, - uses_region_group_mapping: bool | None = None, - ) -> np.ndarray: - """Compute NIXL descriptor IDs for given block IDs.""" - num_ssm_regions = 0 - if self._has_mamba: - assert self._conv_decomp is not None - # NIXL regions per SSM layer = conv sub-projections + 1 SSM temporal - # (Mamba2/GDN: 3+1=4; Mamba1: 1+1=2). - ssm_regions_per_layer = len(self._conv_decomp.local_conv_offsets) + 1 - # Count only the regions that actually hold SSM state; must match - # the descriptors emitted by _build_mamba_local. - num_ssm_regions = ( - len(self._ssm_region_indices or self.block_len_per_layer) - * ssm_regions_per_layer - ) - - num_blocks = dst_num_blocks - if block_size_ratio is not None: - num_blocks = int(num_blocks * block_size_ratio) - use_layer_name_routing = bool(self._transfer_layer_group_ids) - num_desc_regions = ( - len(self._transfer_layer_group_ids) - if use_layer_name_routing - else self.num_regions or len(self.block_len_per_layer) - ) - if region_num_blocks is None: - region_num_blocks = [num_blocks] * num_desc_regions - elif block_size_ratio is not None: - region_num_blocks = [ - int(count * block_size_ratio) for count in region_num_blocks - ] - if region_num_blocks and not use_layer_name_routing: - num_desc_regions = len(region_num_blocks) - assert len(region_num_blocks) == num_desc_regions - region_offsets = np.cumsum([0, *region_num_blocks[:-1]]) - num_fa_descs = sum(region_num_blocks) - - if use_layer_name_routing: - # Descriptors in layer order, using each layer's KV-cache group. - return np.concatenate( - [ - np.asarray(block_ids[g], dtype=np.int32) + region_offsets[k] - for k, g in enumerate(self._transfer_layer_group_ids) - ] - ) - - # All-attention fast path. - if num_ssm_regions == 0: - if region_group_ids is None: - region_group_ids = self.region_group_ids - if uses_region_group_mapping is None: - uses_region_group_mapping = self._uses_region_group_mapping - if not region_group_ids and num_desc_regions == 1: - region_group_ids = [0] - assert len(region_group_ids) == num_desc_regions - if uses_region_group_mapping is None: - uses_region_group_mapping = len(set(region_group_ids)) > 1 - region_group_ids_array = np.asarray(region_group_ids, dtype=np.int32) - # NOTE (NickLucche) With HMA, every kv group has the same number of layers - # and layers from different groups share the same kv tensor. - # eg block_ids=[[1, 2], [3]]->blocks [1, 2] need to be - # read across all regions, same for [3], but group0-group1 blocks will - # always differ (different areas). Therefore we can just flatten the - # block_ids and compute the descs ids for all groups at once. - if not uses_region_group_mapping: - block_arr = np.concatenate( - [np.asarray(group, dtype=np.int32) for group in block_ids] - )[None, :] - return (region_offsets[:, None] + block_arr).ravel() - desc_ids = [] - for group_id, group in enumerate(block_ids): - if not group: - continue - region_ids = np.flatnonzero( - (region_group_ids_array == group_id) - | (region_group_ids_array == _SHARED_REGION_GROUP_ID) - )[:, None] - assert region_ids.size > 0 - group_blocks = np.asarray(group, dtype=np.int32)[None, :] - desc_ids.append((region_offsets[region_ids] + group_blocks).ravel()) - if not desc_ids: - return np.array([], dtype=np.int64) - return np.concatenate(desc_ids) - - # Compute desc ids per group using the right stride: FA descs have - # num_blocks entries per region (kernel granularity, expanded by - # block_size_ratio for heterogeneous block sizes), SSM descs have - # logical_blocks entries per region (no kernel splitting, and never - # ratio-expanded since state blocks are indivisible). - logical_blocks = dst_num_blocks // physical_blocks_per_logical - all_descs: list[np.ndarray] = [] - for i, group in enumerate(block_ids): - group_arr = np.asarray(group, dtype=np.int32) - spec_type = self._group_spec_types[i] - if _is_attention_spec(spec_type): - # A scratch cache lives only in its own regions; every other - # attention group spans all of them. - fa_region_ids = ( - np.asarray(self._scratch_region_indices, dtype=np.int32) - if spec_type is CircularBufferSpec - else np.arange(self.num_regions, dtype=np.int32) - )[:, None] - all_descs.append( - (fa_region_ids * num_blocks + group_arr[None, :]).ravel() - ) - elif _is_ssm_spec(spec_type): - # NOTE (NickLucche) SSM and Attention block regions can - # be exchanged arbitrarily by manager. Therefore, descs - # are laid out as: - # [descs_fa (all regions) | descs_ssm (all regions)]. - # num_fa_descs offset must be computed per-engine since - # P and D can have different num_blocks (and thus - # different FA desc counts). - ssm_region_ids = ( - np.arange(num_ssm_regions, num_ssm_regions + 1, dtype=np.int32) - if i == self._ple_group_index - else np.arange(num_ssm_regions, dtype=np.int32) - )[:, None] - all_descs.append( - ( - ssm_region_ids * logical_blocks - + group_arr[None, :] - + num_fa_descs - ).ravel() - ) - else: - raise ValueError( - f"Unknown spec type {self._group_spec_types[i]} at index {i}" - ) - - return np.concatenate(all_descs) - - def _build_local_splits_from_plan( - self, - plan: TPMapping, - src_blocks_data: np.ndarray, - num_fa_descs: int, - block_size_ratio: int = 1, - ) -> Iterator[np.ndarray]: - """Build split handle data for P_TP > D_TP scenario. - - num_fa_descs is the boundary between FA and SSM descriptors. - Split counts are derived from source_ranks_per_group lengths. - FA uses rank_to_attention_slot for the slot offset; - SSM uses the rank's positional index. - Each Nx5 strided desc covers one region, so slicing it slices every block. - """ - fa_idx = next( - i for i, t in enumerate(self._group_spec_types) if _is_attention_spec(t) - ) - fa_num_splits = len(plan.source_ranks_per_group[fa_idx]) - - starts = self._strided_desc_starts(src_blocks_data) - num_descs = int(src_blocks_data[:, 4].sum()) - has_ssm_descs = num_fa_descs < num_descs - ssm_idx = next( - (i for i, t in enumerate(self._group_spec_types) if _is_ssm_spec(t)), - None, - ) - ssm_num_splits = ( - len(plan.source_ranks_per_group[ssm_idx]) - if has_ssm_descs and ssm_idx is not None - else 0 - ) - sharded_desc_end = num_descs - ( - self._logical_num_blocks if self._ple_region_index is not None else 0 - ) - assert num_fa_descs <= sharded_desc_end - - # REPLICATE (MLA) descs are read whole on every rank; SPLIT (full-attn) - # descs take this rank's head slice. SSM descs before the PLE pages are - # sharded by the rank's positional index; PLE pages are copied whole. - is_fa = starts < num_fa_descs - fa_replicated = self._fa_desc_replicated( - starts[is_fa], num_fa_descs, block_size_ratio - ) - assert block_size_ratio == 1 or fa_num_splits == 1 or fa_replicated.all(), ( - "Head-sharded attention reads with P_TP > D_TP and heterogeneous " - "block sizes are not supported" - ) - fa_split = is_fa.copy() - fa_split[is_fa] = ~fa_replicated - ssm_split = ~is_fa & (starts < sharded_desc_end) - - for p_idx, p_rank in enumerate(plan.all_source_ranks): - fa_slot = plan.rank_to_attention_slot.get(p_rank, 0) - handle = src_blocks_data.copy() - handle[fa_split, 1] //= fa_num_splits - handle[fa_split, 0] += fa_slot * handle[fa_split, 1] - if ssm_num_splits: - handle[ssm_split, 1] //= ssm_num_splits - handle[ssm_split, 0] += p_idx * handle[ssm_split, 1] - yield handle - - def _needs_split_local_xfer_handles(self, tp_ratio: int, plan: TPMapping) -> bool: - """Whether reads need per-source slices of the local KV region. - - Pure MLA attention is replicated across TP ranks and writes the whole - local region. Multiple physical remote workers may still participate - because DCP assigns them disjoint blocks, but that does not require - splitting the local region. Hybrid MLA+SSM is different: its mapping - contains multiple source ranks for the sharded SSM state. - """ - return tp_ratio < 0 and (not self.use_mla or len(plan.all_source_ranks) > 1) - - def _fa_desc_replicated( - self, - desc_starts: np.ndarray, - num_fa_descs: int, - block_size_ratio: int = 1, - ) -> np.ndarray: - """Replicate flag per FA strided desc, given each one's flat block index. - - _build_fa_local emits regions in order, with - ``region_num_blocks[i] * block_size_ratio`` blocks per region. A strided - desc never spans regions, so the region owning its first block decides. - """ - assert self.transfer_topo is not None - n_regions = len(self.block_len_per_layer) - if n_regions == 0 or self.num_regions == 0: - return np.zeros(len(desc_starts), dtype=bool) - region_indices = self._transfer_layer_region_indices or range(n_regions) - region_ends = np.cumsum( - [self.region_num_blocks[i] * block_size_ratio for i in region_indices] - ) - assert region_ends[-1] == num_fa_descs, ( - f"FA region blocks {region_ends[-1]} != num_fa_descs {num_fa_descs}" - ) - replicated = np.array([self._is_region_replicated(i) for i in region_indices]) - return replicated[np.searchsorted(region_ends, desc_starts, side="right")] - - def _is_region_replicated(self, region_idx: int) -> bool: - """Whether region ``region_idx`` is transferred REPLICATE vs SPLIT. - - REPLICATE (MLA): identical on every rank, whole block read from one - rank at offset 0, key-only. SPLIT (full-attn): head-sharded across TP. - Defaults to SPLIT when the per-region map is unset (e.g. tests that set - block_len_per_layer without register_kv_caches). - """ - return region_idx < len(self._region_is_mla) and self._region_is_mla[region_idx] - - def _set_region_layers(self, region_layers: list[list[str]]) -> None: - layer_names = [layer_name for region in region_layers for layer_name in region] - assert len(layer_names) == len(set(layer_names)), ( - "A KV cache layer spans multiple NIXL regions" - ) - self.region_members: list[list[str]] = region_layers - if not self._requires_layer_name_routing(): - return - group_by_layer = self.kv_cache_config.transfer_group_index_by_layer - ungrouped = [name for name in layer_names if name not in group_by_layer] - assert not ungrouped, f"KV cache layers outside any local group: {ungrouped}" - self._transfer_layer_names = tuple(layer_names) - self._transfer_layer_region_indices = tuple( - region_idx - for region_idx, region in enumerate(region_layers) - for _ in region - ) - self._transfer_layer_group_ids = tuple( - group_by_layer[name] for name in layer_names - ) - - def _tracks_region_layers(self) -> bool: - """Whether regions advertise their member layer names. - - Push workers do so for HMA and packed layouts, so a PP producer can - match layers by name instead of by region index. - """ - return ( - self._supports_pp_hma - and (self._is_hma_required or self._has_packed_cache) - and not self._has_mamba - ) - - def _requires_layer_name_routing(self) -> bool: - """Whether PP push must match HMA/packed layers by name, not region index.""" - return self.pp_size > 1 and self._tracks_region_layers() - - def _align_remote_regions_by_layer( - self, nixl_agent_meta: NixlAgentMetadata - ) -> None: - """Select remote regions for this PP stage in local layer order.""" - remote_region_layers = nixl_agent_meta.region_members - # Region-index routing here would silently transfer stale KV. - assert remote_region_layers, "Remote advertised no region_members" - assert ( - len(nixl_agent_meta.kv_caches_base_addr) - == len(nixl_agent_meta.block_lens) - == len(nixl_agent_meta.block_strides) - == len(remote_region_layers) - ), "Remote region metadata lengths disagree" - assert all( - values is None or len(values) == len(remote_region_layers) - for values in ( - nixl_agent_meta.region_num_blocks, - nixl_agent_meta.region_group_ids, - nixl_agent_meta.region_names, - nixl_agent_meta.region_mem_types, - ) - ), "Remote region metadata lengths disagree" - - remote_region_by_layer: dict[str, int] = {} - for region_idx, layer_names in enumerate(remote_region_layers): - for layer_name in layer_names: - assert layer_name not in remote_region_by_layer, ( - f"Remote advertised layer {layer_name!r} in multiple regions" - ) - remote_region_by_layer[layer_name] = region_idx - - missing = [ - name - for name in self._transfer_layer_names - if name not in remote_region_by_layer - ] - assert not missing, f"Remote is missing locally owned layers: {missing}" - - packed_regions = { - remote_region_by_layer[name] - for name in nixl_agent_meta.packed_member_layouts - } - remote_regions = [ - remote_region_by_layer[name] for name in self._transfer_layer_names - ] - nixl_agent_meta.kv_caches_base_addr = [ - nixl_agent_meta.kv_caches_base_addr[i] for i in remote_regions - ] - nixl_agent_meta.block_lens = [ - nixl_agent_meta.block_lens[i] for i in remote_regions - ] - nixl_agent_meta.block_strides = [ - nixl_agent_meta.block_strides[i] for i in remote_regions - ] - if nixl_agent_meta.region_num_blocks is not None: - nixl_agent_meta.region_num_blocks = [ - nixl_agent_meta.region_num_blocks[i] for i in remote_regions - ] - if nixl_agent_meta.region_group_ids is not None: - nixl_agent_meta.region_group_ids = [ - nixl_agent_meta.region_group_ids[i] for i in remote_regions - ] - if nixl_agent_meta.region_mem_types is not None: - nixl_agent_meta.region_mem_types = [ - nixl_agent_meta.region_mem_types[i] for i in remote_regions - ] - if nixl_agent_meta.region_names is not None: - nixl_agent_meta.region_names = list(self._transfer_layer_names) - if nixl_agent_meta.packed_member_layouts: - for i, layer_name in enumerate(self._transfer_layer_names): - if remote_regions[i] not in packed_regions: - continue - assert layer_name in nixl_agent_meta.packed_member_layouts, ( - f"Remote packed layer {layer_name!r} has no layout" - ) - offset, page_size = nixl_agent_meta.packed_member_layouts[layer_name] - assert ( - 0 <= offset < offset + page_size <= nixl_agent_meta.block_lens[i] - ), f"Remote packed layer {layer_name!r} escapes its block" - local_region = self._transfer_layer_region_indices[i] - assert page_size == self.block_len_per_layer[local_region], ( - f"Packed MLA page sizes must match for layer {layer_name!r}" - ) - nixl_agent_meta.kv_caches_base_addr[i] += offset - nixl_agent_meta.block_lens[i] = page_size - nixl_agent_meta.packed_member_layouts = {} - # One layer per region keeps a second alignment pass a no-op. - nixl_agent_meta.region_members = [[name] for name in self._transfer_layer_names] - - def _split_remote_packed_rows(self, nixl_agent_meta: NixlAgentMetadata) -> None: - """Map the remote's whole-row region onto our per-layer regions. - - E.g. a BLHNC prefiller registers one region whose block is a full row - holding every layer's page, while a HiSparse decoder registers one - region per layer. Using the advertised `packed_member_layouts`, the - remote row is rewritten in place as one virtual region per local layer: - - remote row b: [ L0 page | L1 page | ... ] base=B, stride=S - ^off0 ^off1 - - remote metadata, before: [(B, S, S)] # (base, len, stride) - remote metadata, after: [(B + off0, page0, S), # L0 - (B + off1, page1, S)] # L1 - - Remote regions outside the row (e.g. a separate indexer) pass through. - """ - layouts = nixl_agent_meta.packed_member_layouts - remote_names = nixl_agent_meta.region_names - assert remote_names is not None, "Remote advertised no region_names" - (row,) = [i for i, name in enumerate(remote_names) if name in layouts] - if nixl_agent_meta.region_group_ids is not None and ( - nixl_agent_meta.region_group_ids[row] == _SHARED_REGION_GROUP_ID - ): - raise NotImplementedError( - "Reading multi-group packed KV rows into per-layer regions is not " - "supported; use the same KV cache layout on P and D." - ) - remote_index = {name: i for i, name in enumerate(remote_names)} - regions = [ - (row, *layouts[name]) - if name in layouts - else (remote_index[name], 0, nixl_agent_meta.block_lens[remote_index[name]]) - for name in self.region_names - ] - nixl_agent_meta.kv_caches_base_addr = [ - nixl_agent_meta.kv_caches_base_addr[i] + offset for i, offset, _ in regions - ] - nixl_agent_meta.block_lens = [page_size for _, _, page_size in regions] - nixl_agent_meta.block_strides = [ - nixl_agent_meta.block_strides[i] for i, _, _ in regions - ] - for attr in ("region_num_blocks", "region_group_ids", "region_mem_types"): - values = getattr(nixl_agent_meta, attr) - if values is not None: - setattr(nixl_agent_meta, attr, [values[i] for i, _, _ in regions]) - nixl_agent_meta.region_names = list(self.region_names) - nixl_agent_meta.packed_member_layouts = {} - - def __init__( - self, - vllm_config: "VllmConfig", - engine_id: str, - kv_cache_config: "KVCacheConfig", - ): - nixl_wrapper_cls = NixlWrapper - if nixl_wrapper_cls is None: - logger.error("NIXL is not available") - raise RuntimeError("NIXL is not available") - # Descriptor lists are prepared in NIXL's strided (Nx5) form. - check_nixl_version("1.5.0") - logger.info("Initializing NIXL wrapper") - logger.info("Initializing NIXL worker %s", engine_id) - - # Config. - self.vllm_config = vllm_config - - if vllm_config.kv_transfer_config is None: - raise ValueError("kv_transfer_config must be set for NixlConnector") - self.kv_transfer_config = vllm_config.kv_transfer_config - - self.nixl_backends = vllm_config.kv_transfer_config.get_from_extra_config( - "backends", ["UCX"] - ) - kv_lease_duration: int = vllm_config.kv_transfer_config.get_from_extra_config( - "kv_lease_duration", 30 - ) - # NOTE (NickLucche): For now we use a hardcoded value for a simpler interface. - self._lease_extension = kv_lease_duration * 2 // 3 - - self._bidirectional_kv_xfer_enabled: bool = ( - vllm_config.kv_transfer_config.get_from_extra_config( - "bidirectional_kv_xfer", False - ) - ) - - self.kv_cache_config = kv_cache_config - transfer_block_sizes = [ - group.kv_cache_spec.block_size - for group in kv_cache_config.transfer_groups - if get_representative_spec_type(group.kv_cache_spec) - not in (MambaSpec, CircularBufferSpec) - ] - self.block_size = ( - math.lcm(*transfer_block_sizes) - if transfer_block_sizes - else cast(int, vllm_config.cache_config.block_size) - ) - # Per-layer specs, unwrapping UniformTypeKVCacheSpecs group wrappers. - self._layer_specs: dict[str, KVCacheSpec] = {} - for group in kv_cache_config.transfer_groups: - group_spec = group.kv_cache_spec - if isinstance(group_spec, UniformTypeKVCacheSpecs): - self._layer_specs.update(group_spec.kv_cache_specs) - else: - self._layer_specs.update(dict.fromkeys(group.layer_names, group_spec)) - self._is_hma_required = ( - not vllm_config.scheduler_config.disable_hybrid_kv_cache_manager - and any( - not isinstance(spec, FullAttentionSpec) - for spec in self._layer_specs.values() - ) - ) - - # ---- Model state (derived from model config) ---- - mamba_ssm_size = (0, 0) - # Conv state sub-projection decomposition (None when no Mamba). - # The transfer requires DS (dim, state_len) conv layout so that - # conv sub-projections are contiguous in memory. - self._conv_decomp: MambaConvSplitInfo | None = None - self._has_mamba = any( - isinstance(spec, MambaSpec) for spec in self._layer_specs.values() - ) - self._is_csa_linear = any( - get_representative_spec_type(group.kv_cache_spec) is CircularBufferSpec - for group in kv_cache_config.kv_cache_groups - ) - self._ple_group_index: int | None = None - if self._is_csa_linear: - ple_groups = [ - (index, group) - for index, group in enumerate(kv_cache_config.kv_cache_groups) - if group.layer_names - and all( - isinstance(spec, MambaSpec) and spec.tp_replicated - for spec in iter_layer_specs(group.kv_cache_spec) - ) - ] - if len(ple_groups) > 1 or any( - len(group.layer_names) != 1 for _, group in ple_groups - ): - raise ValueError( - "CSA-linear NIXL requires at most one PLE cache owner." - ) - self._ple_group_index = ple_groups[0][0] if ple_groups else None - - if self._has_mamba: - assert self._is_hma_required - from vllm.model_executor.layers.mamba.mamba_utils import ( - is_conv_state_dim_first, - ) - - assert is_conv_state_dim_first(), ( - "3-read Mamba conv transfer requires DS conv state layout. " - "Set VLLM_SSM_CONV_STATE_LAYOUT=DS" - ) - mamba_spec = next( - spec - for spec in self._layer_specs.values() - if isinstance(spec, MambaSpec) - and (not self._is_csa_linear or not spec.tp_replicated) - ) - self._conv_decomp = derive_mamba_conv_split( - mamba_spec, - vllm_config.parallel_config.tensor_parallel_size, - ) - mamba_ssm_size = self._conv_decomp.ssm_sizes - self._mamba_ssm_size = mamba_ssm_size - - # Agent. - non_ucx_backends = [b for b in self.nixl_backends if b != "UCX"] - # Configure NIXL num_threads to avoid UAR exhaustion on Mellanox NICs. - # Each UCX thread allocates UARs (doorbell pages) via DevX, and - # excessive NIXL UAR usage can exhaust NIC UAR space. This can cause - # components like NVSHMEM (used by DeepEP kernels) to fail during RDMA - # initialization with "mlx5dv_devx_alloc_uar" errors. - # Ref: https://network.nvidia.com/files/doc-2020/ethernet-adapters-programming-manual.pdf#page=63 - num_threads = vllm_config.kv_transfer_config.get_from_extra_config( - "num_threads", 4 - ) - if nixl_agent_config is None: - config = None - else: - # Enable telemetry by default for NIXL 0.7.1 and above. - config = ( - nixl_agent_config(backends=self.nixl_backends, capture_telemetry=True) - if len(non_ucx_backends) > 0 - else nixl_agent_config(num_threads=num_threads, capture_telemetry=True) - ) - - self.nixl_wrapper = nixl_wrapper_cls(str(uuid.uuid4()), config) - # Map of engine_id -> {(pp_rank, tp_rank): agent_name, ...}. - # non-PP remote uses pp_rank 0, i.e. (0, tp_rank). - self._remote_agents: dict[EngineId, dict[tuple[int, int], str]] = defaultdict( - dict - ) - # Failed reads need a local metadata probe; invalid peers wait for drain. - self._failed_remote_engines: set[EngineId] = set() - self._invalid_remote_engines: set[EngineId] = set() - # Invalid engine -> (backoff, earliest release time). Kept across release - # so that repeat invalidations back off. - self._invalid_engine_backoff: dict[EngineId, tuple[float, float]] = {} - # Map of engine_id -> clock offset. - self._engine_clock_offset: dict[EngineId, float] = {} - - # Metadata. - self.engine_id: EngineId = engine_id - self.tp_rank = get_tensor_model_parallel_rank() - self.world_size = get_tensor_model_parallel_world_size() - self.pcp_size = vllm_config.parallel_config.prefill_context_parallel_size - self.dcp_size = vllm_config.parallel_config.decode_context_parallel_size - self.pcp_rank = get_pcp_group().rank_in_group if self.pcp_size > 1 else 0 - - # TP1 PCP+DCP owns distinct KV shards; replicated PCP uses rank zero. - self.pcp_dcp_sharded = self.pcp_size > 1 and self.dcp_size > 1 - self.transfer_tp_size = ( - self.pcp_size if self.pcp_dcp_sharded else self.world_size - ) - self.transfer_tp_rank = self.pcp_rank if self.pcp_dcp_sharded else self.tp_rank - - # MLA is either fully replicated or fully sharded across transfer ranks. - self.dcp_rank = self.transfer_tp_rank % self.dcp_size - - self.num_blocks = kv_cache_config.num_blocks - self.enable_permute_local_kv = False - self.enable_heterogeneous_attn_post_process = False - - # KV Caches and nixl tracking data. - self.device_type = current_platform.device_type - self.kv_buffer_device: str = vllm_config.kv_transfer_config.kv_buffer_device - if self.device_type not in _NIXL_SUPPORTED_DEVICE: - raise RuntimeError(f"{self.device_type} is not supported.") - elif self.kv_buffer_device not in _NIXL_SUPPORTED_DEVICE[self.device_type]: - raise RuntimeError( - f"{self.device_type} with {self.kv_buffer_device} kv_buffer " - "is not supported." - ) - self.device_kv_caches: dict[str, torch.Tensor] = {} - - # cpu kv buffer for xfer - # used when device memory can not be registered under nixl - self.host_xfer_buffers: dict[str, torch.Tensor] = {} - if self.device_type == "cpu": - self.use_host_buffer = False - else: - self.use_host_buffer = self.kv_buffer_device == "cpu" - - # reserve different cores for start_load_kv() from model_forward() - if self.device_type == "cpu": - numa_core_list = current_platform.discover_numa_topology() - # setup one last core in each numa for kv transfer. - rsv_cores_for_kv = [ - max(each_numa_core_list) for each_numa_core_list in numa_core_list - ] - - if rsv_cores_for_kv: - if not hasattr(os, "sched_setaffinity"): - raise NotImplementedError( - "os.sched_setaffinity is not available on this platform" - ) - os.sched_setaffinity(0, rsv_cores_for_kv) - - # support for oot platform which can't register nixl memory - # type based on kv_buffer_device - nixl_memory_type = current_platform.get_nixl_memory_type() - if nixl_memory_type is None: - if self.kv_buffer_device in ["cuda", "xpu"]: - nixl_memory_type = "VRAM" - elif self.kv_buffer_device == "cpu": - nixl_memory_type = "DRAM" - if nixl_memory_type is None: - raise RuntimeError( - f"{self.device_type} with {self.kv_buffer_device} kv_buffer " - "is not supported." - ) - self.nixl_memory_type = nixl_memory_type - - # Note: host xfer buffer ops when use_host_buffer is True - self.copy_blocks: CopyBlocksOp | None = None - - # Map of engine_id -> kv_caches_base_addr. For TP case, each local - self.device_id: int = 0 - # Current rank may pull from multiple remote TP workers. - # EngineId, dict[int, list[int]] -> engine_id, tp_rank, base_addr_for_layer - self.kv_caches_base_addr = defaultdict[EngineId, dict[int, list[int]]](dict) - - # Number of NIXL regions. Currently one region per cache - # (so 1 per layer for MLA, otherwise 2 per layer) - self.num_regions = 0 - self.region_mem_types: list[str] = [] - self.region_group_ids: list[int] = [] - self._uses_region_group_mapping = False - self.region_names: list[str] = [] - self.region_num_blocks: list[int] = [] - self._mixed_mem_types = False - # True on TP ranks that share a HiSparse host pool with TP rank 0 and - # leave its NIXL registration and DRAM reads to rank 0. - self._skip_dram_xfer = False - self._desc_is_dram_by_block_size: dict[int, np.ndarray] = {} - self._desc_pos_by_block_size: dict[int, np.ndarray] = {} - self._dram_src_handles_by_block_size: dict[int, int | None] = {} - - # PP>1 (push mode): this worker holds a contiguous layer slice and - # transfers into the matching sub-range of a PP=1 remote's regions. - self.pp_size = vllm_config.parallel_config.pipeline_parallel_size - self._remote_region_offset = 0 - if self.pp_size > 1: - if self._is_hma_required and not self._supports_pp_hma: - raise NotImplementedError( - "NixlConnector (pull) does not support pipeline_parallel_size " - "> 1 with hybrid KV cache layouts (HMA); use NixlPushConnector " - "for PP + HMA." - ) - # PP push routes HMA (hybrid) attention layouts by layer name; - # Mamba/SSM hybrids are not yet supported under PP. - if self._has_mamba: - raise NotImplementedError( - "NixlPushConnector does not support pipeline_parallel_size > 1 " - "with Mamba/SSM hybrid KV cache layouts yet." - ) - # Decode-side PP is unsupported (completions counted per consumer rank). - if vllm_config.kv_transfer_config.kv_role == "kv_consumer": - raise NotImplementedError( - "NixlPushConnector consumer (decode) does not support " - "pipeline_parallel_size > 1." - ) - # Keep heartbeat handshakes to a PP-sharded producer notif-only. - self._hb_handshake_notif_only = False - - # nixl_prepped_dlist_handle. - self.src_xfer_handles_by_block_size: dict[int, int] = {} - # Local descriptor arrays per remote block size (block_size_ratio>1), - # kept for building per-tp-ratio splits at the same granularity. - self.src_blocks_data_by_block_size: dict[int, np.ndarray] = {} - # Populated dynamically during handshake based on remote configuration. - # Per-source split handles, keyed by (tp_ratio, remote_block_size). - self.src_xfer_handles_by_tp_ratio: dict[tuple[int, int], list[int]] = {} - self._dram_src_handles_by_tp_ratio: dict[tuple[int, int], list[int | None]] = {} - # Map of engine_id -> {tp_rank: nixl_prepped_dlist_handle (int)}. - self.dst_xfer_side_handles = defaultdict[EngineId, dict[int, int]](dict) - - # Map of engine_id -> num_blocks. All ranks in the same deployment will - # have the same number of blocks. - self.dst_num_blocks: dict[EngineId, int] = {} - self.dst_region_num_blocks: dict[EngineId, list[int]] = {} - self.dst_region_group_ids: dict[EngineId, list[int]] = {} - self.dst_uses_region_group_mapping: dict[EngineId, bool] = {} - self.dst_region_mem_types: dict[EngineId, list[str]] = {} - self._registered_descs: list[Any] = [] - - # In progress transfers. - # [req_id -> list[handle]] - self._recving_metadata: dict[ReqId, ReqMeta] = {} - self._recving_transfers = defaultdict[ReqId, list[TransferHandle]](list) - # Track the expiration time of requests that are waiting to be sent. - self._reqs_to_send: dict[ReqId, float] = {} - # Replicated-KV PCP ranks > 0 never transfer; requests they are asked - # to send are reported finished immediately (see get_finished). - self._replicated_pcp_done_sending: set[ReqId] = set() - # Set of requests that have been part of a batch, regardless of status. - self._reqs_to_process: set[ReqId] = set() - - # Invalid blocks from failed NIXL operations (thread-safe queue of block ids) - self._invalid_block_ids: queue.Queue[set[int]] = queue.Queue() - # requests that skipped transfer (handshake or transfer failures) - # Uses Queue for thread-safe cross-thread coordination with the - # background handshake thread, matching the _ready_requests pattern. - self._failed_recv_reqs: queue.Queue[ReqId] = queue.Queue() - self._recv_failures: set[ReqId] = set() - self._pending_recv_notifs: dict[ReqId, list[tuple[str, bytes]]] = {} - - # Handshake metadata of this worker for NIXL transfers. - self.xfer_handshake_metadata: NixlHandshakePayload | None = None - # Background thread for initializing new NIXL handshakes. - self._handshake_initiation_executor = ThreadPoolExecutor( - # NIXL is not guaranteed to be thread-safe, limit 1 worker. - max_workers=1, - thread_name_prefix="vllm-nixl-handshake-initiator", - ) - self._ready_requests = queue.Queue[tuple[ReqId, ReqMeta]]() - self._handshake_futures: dict[ - EngineId, Future[tuple[dict[tuple[int, int], str], float]] - ] = {} - # Protects _handshake_futures and _remote_agents. - self._handshake_lock = threading.RLock() - - # TTL-based eviction of stale remote engine state. - self._engine_last_active: dict[EngineId, float] = {} - self._engine_ttl: float = vllm_config.kv_transfer_config.get_from_extra_config( - "engine_ttl", 3600.0 - ) - # Map of pull peer (host, port) -> engine_id currently serving it. - self._engine_by_address: dict[tuple[str, int], EngineId] = {} - - self.model_config = vllm_config.model_config - - self.use_mla = self.model_config.use_mla - - # Get the attention backend from the first layer - # NOTE (NickLucche) models with multiple backends are not supported yet - self.attn_backends = get_current_attn_backends(vllm_config) - self.backend_name = self.attn_backends[0].get_name() - - self.kv_cache_layout = ( - vllm_config.cache_config.get_resolved_kv_cache_layout().name - ) - self.host_buffer_kv_cache_layout = self.kv_cache_layout - logger.info( - "Detected attention backend(s) %s", - [backend.get_name() for backend in self.attn_backends], - ) - logger.info("Detected kv cache layout %s", self.kv_cache_layout) - - # lazy initialized in register_kv_caches - self.compat_hash: str | None = None - self.transfer_topo: TransferTopology | None = None - - # With heterogeneous TP (or DCP), P must wait for all assigned D - # workers to finish reading before safely freeing the blocks. - self.consumer_notification_counts_by_req = defaultdict[ReqId, int](int) - self.expected_consumer_notifications_by_req: dict[ReqId, int] = {} - self.xfer_stats = NixlKVConnectorStats() - - self._physical_blocks_per_logical_kv_block = 1 - self._sync_block_size_with_kernel() - - # Unwrap UniformTypeKVCacheSpecs to get the representative spec type - self._group_spec_types = tuple( - get_representative_spec_type(g.kv_cache_spec) - for g in self.kv_cache_config.transfer_groups - ) - # Per-region MLA flag, 1:1 with block_len_per_layer. True -> REPLICATE - # (MLA), False -> SPLIT (head-sharded full-attn). Mixed only for models - # combining both (e.g. GQA main + MLA Eagle-3 draft). - self._region_is_mla = list[bool]() - self._ssm_region_indices = list[int]() - # Regions holding a scratch cache (the CSA compressor circular buffer). - # Tracked explicitly because a scratch page shares its address with the - # page it overlays, so its regions cannot be recovered from spec type. - self._scratch_region_indices = list[int]() - self._ple_region_index: int | None = None - # Page length of the PLE short-conv state; its region may be shared with - # pages of a different (smaller) size, so it cannot use the region's. - self._ple_block_len: int | None = None - - # Enable different block lengths for different layers *only* when MLA is used. - # This is not used for SSM layers, which use the counterpart `mamba_ssm_size`. - self.block_len_per_layer = list[int]() - - # Per-region block stride in bytes. Taken from the registered tensor's - # stride(0) so it stays correct under layouts that interleave layers - # within a block (BLHNC/BHLNC), where stride > block_len. - self.block_stride_per_layer = list[int]() - - # Block-outermost layouts (BLHNC/BHLNC) pack every layer's page into one - # block row of a shared allocation. - self._has_packed_cache = KVCacheLayout[self.kv_cache_layout].is_block_outermost - # Region layer membership is populated for HMA and packed layouts. - self.region_members = [] - self._packed_member_layouts: dict[str, tuple[int, int]] = {} - # Local layer order, set by ``_set_region_layers`` only when - # routing by layer name. Empty means region-index routing. - self._transfer_layer_names = tuple[str, ...]() - self._transfer_layer_region_indices = tuple[int, ...]() - self._transfer_layer_group_ids = tuple[int, ...]() - # Per-engine TP mappings. Generated during handshake. - self.tp_mappings: dict[EngineId, TPMapping] = {} - - self.enforce_compat_hash = self.kv_transfer_config.get_from_extra_config( - "enforce_handshake_compat", True - ) - - def _validate_remote_parallel_config( - self, agent_metadata: NixlAgentMetadata - ) -> None: - local_pcp_size = self.pcp_size - local_dcp_size = self.dcp_size - remote_pcp_size = agent_metadata.pcp_size - remote_dcp_size = agent_metadata.dcp_size - if remote_pcp_size > 1 and remote_dcp_size not in (1, remote_pcp_size): - raise NotImplementedError( - "Remote NixlConnector PCP+DCP does not span the full PCP group. " - f"Remote PCP/DCP={remote_pcp_size}/{remote_dcp_size}." - ) - if (local_pcp_size > 1 and local_dcp_size == 1 and remote_dcp_size > 1) or ( - remote_pcp_size > 1 and remote_dcp_size == 1 and local_dcp_size > 1 - ): - raise NotImplementedError( - "Replicated PCP cannot be paired with a DCP-sharded NIXL peer. " - f"Local PCP/DCP={local_pcp_size}/{local_dcp_size}; " - f"remote PCP/DCP={remote_pcp_size}/{remote_dcp_size}." - ) - - def _sync_block_size_with_kernel(self) -> None: - backends, specs = get_current_attn_backends_and_specs( - self.vllm_config, self.kv_cache_config, self.attn_backends - ) - kernel_block_size = select_common_block_size(self.block_size, backends, specs) - # Number of blocks not accounting for kernel block mismatches - self._logical_num_blocks = self.num_blocks - if self.block_size != kernel_block_size: - logger.info_once( - "User-specified logical block size (%s) does not match" - " physical kernel block size (%s). Using the latter.", - self.block_size, - kernel_block_size, - ) - assert self.block_size > kernel_block_size - self._physical_blocks_per_logical_kv_block = ( - self.block_size // kernel_block_size - ) - self.block_size = kernel_block_size - self.num_blocks *= self._physical_blocks_per_logical_kv_block - - def _validate_csa_linear_tp_layout(self, remote_tp_size: int) -> None: - """Reject P/D pairs whose main-KV pages have different head layouts. - - A rank at TP < total KV heads holds several heads per page, while a - rank at TP >= total KV heads holds exactly one (replicated beyond - that). CSA-linear pages are shared tensors that transfer as whole - block units, so endpoints on opposite sides of that boundary lay - them out differently and cannot exchange them. - """ - assert self.transfer_topo is not None - total_kv_heads = self.transfer_topo.total_num_kv_heads - if (self.world_size < total_kv_heads) != (remote_tp_size < total_kv_heads): - raise ValueError( - "CSA-linear NIXL requires both endpoints on the same side of " - f"the KV-head sharding boundary ({total_kv_heads} KV heads): " - f"got local TP {self.world_size}, remote TP {remote_tp_size}." - ) - - def _nixl_handshake( - self, - host: str, - port: int, - remote_tp_size: int, - expected_engine_id: str, - remote_dcp_size: int = 1, - remote_pp_size: int = 1, - notif_agents_only: bool = False, - ) -> tuple[dict[tuple[int, int], str], float]: - """Do a NIXL handshake with a remote instance.""" - if self._is_csa_linear: - self._validate_csa_linear_tp_layout(remote_tp_size) - - # the first time we connect to a remote agent. - # be careful, the handshake happens in a background thread. - # it does not have an active cuda context until any cuda runtime - # call is made. when UCX fails to find a valid cuda context, it will - # disable any cuda ipc communication, essentially disabling any NVLink - # communication. - # when we are using device buffers, we need to set the device - # explicitly to make sure the handshake background thread has a valid - # cuda context. - if not self.use_host_buffer: - current_platform.set_device(self.device_id) - - # When target instance TP > local TP, we need to perform multiple - # handshakes. Do it in a single background job for simplicity. - # Regardless, only handshake with the remote TP rank(s) that current - # local rank will read from. Note that With homogeneous TP, - # this happens to be the same single rank_i. - assert self.transfer_topo is not None - p_remote_ranks = self.transfer_topo.handshake_target_ranks( - remote_tp_size, remote_dcp_size - ) - remote_rank_to_agent_name: dict[tuple[int, int], str] = {} - path = make_zmq_path("tcp", host, port) - # Clock offset to the peer, estimated from the handshake round-trip. - # Keep the lowest-RTT sample: hop cost is ~uniform across ranks, so a - # higher RTT is just noise that skews the midpoint estimate. - best_rtt = float("inf") - best_offset: float | None = None - - with zmq_ctx(zmq.REQ, path) as sock: - for remote_pp_rank, remote_rank in itertools.product( - range(remote_pp_size), p_remote_ranks - ): - logger.debug( - "Querying metadata on path: %s at remote pp rank %s, tp rank %s", - path, - remote_pp_rank, - remote_rank, - ) - - # Send query for the request. - msg = msgspec.msgpack.encode( - (GET_META_MSG, remote_pp_rank, remote_rank) - ) - # Set receive timeout to 5 seconds to avoid hanging on dead server - sock.setsockopt(zmq.RCVTIMEO, 5000) # milliseconds - start_time = time.perf_counter() - sock.send(msg) - reply_parts = sock.recv_multipart() - recv_time = time.perf_counter() - assert len(reply_parts) == 2 - handshake_bytes = reply_parts[0] - - remote_perf = msgspec.msgpack.decode(reply_parts[1]) - rtt = recv_time - start_time - if rtt < best_rtt: - best_rtt = rtt - best_offset = remote_perf - (start_time + recv_time) / 2 - - # Decode handshake payload to get compatibility hash - handshake_decoder = msgspec.msgpack.Decoder(NixlHandshakePayload) - try: - handshake_payload = handshake_decoder.decode(handshake_bytes) - except (msgspec.DecodeError, msgspec.ValidationError) as e: - raise RuntimeError( - f"Failed to decode NixlHandshakePayload. This likely indicates " - f"an incompatibility between connector version. Error: {e}" - ) from e - - got_metadata_time = time.perf_counter() - logger.debug( - "NIXL handshake: get metadata took: %s", - got_metadata_time - start_time, - ) - - # Check compatibility hash BEFORE decoding agent metadata - assert self.compat_hash is not None - if ( - self.enforce_compat_hash - and handshake_payload.compatibility_hash != self.compat_hash - ): - raise RuntimeError( - f"NIXL compatibility hash mismatch. " - f"Local: {self.compat_hash}, " - f"Remote: {handshake_payload.compatibility_hash}. " - f"Prefill and decode instances have incompatible " - f"configurations. This may be due to: different vLLM versions," - f" models, dtypes, KV cache layouts, attention backends, etc. " - f"Both instances must use identical configurations." - f"Disable this check using " - f'--kv-transfer-config \'{{"kv_connector_extra_config": ' - f'{{"enforce_handshake_compat": false}}}}\'' - ) - - logger.info( - "NIXL compatibility check passed (hash: %s)", - handshake_payload.compatibility_hash, - ) - - # Decode agent metadata - metadata_decoder = msgspec.msgpack.Decoder(NixlAgentMetadata) - try: - metadata = metadata_decoder.decode( - handshake_payload.agent_metadata_bytes - ) - except (msgspec.DecodeError, msgspec.ValidationError) as e: - # This should not happen if hash matched - raise RuntimeError( - f"Failed to decode NixlAgentMetadata. Error: {e}" - ) from e - - self._validate_remote_parallel_config(metadata) - - # Ensure engine id matches. - if metadata.engine_id != expected_engine_id: - raise RuntimeError( - f"Remote NIXL agent engine ID mismatch. " - f"Expected {expected_engine_id}," - f"received {metadata.engine_id}." - ) - - # Register Remote agent. - if notif_agents_only: - remote_agent_name = self._add_notif_only_remote_agent( - metadata, remote_tp_size, metadata.dcp_size - ) - else: - remote_agent_name = self.add_remote_agent( - metadata, remote_rank, remote_tp_size, metadata.dcp_size - ) - setup_agent_time = time.perf_counter() - logger.debug( - "NIXL handshake: add agent took: %s (notif_agents_only=%s)", - setup_agent_time - got_metadata_time, - notif_agents_only, - ) - remote_ranks = (remote_pp_rank, remote_rank) - remote_rank_to_agent_name[remote_ranks] = remote_agent_name - - assert best_offset is not None - return remote_rank_to_agent_name, best_offset - - def _add_notif_only_remote_agent( - self, metadata: NixlAgentMetadata, remote_tp_size: int, remote_dcp_size: int = 1 - ) -> str: - """Load a remote agent for notifs only on the push-mode decode side. - - Skips descriptor setup but records engine info for block accounting. - """ - assert self.transfer_topo is not None - self.transfer_topo.register_remote_engine( - metadata.engine_id, - EngineTransferInfo( - remote_tp_size=remote_tp_size, - remote_block_size=metadata.block_size, - remote_block_len=metadata.block_lens[0], - remote_physical_blocks_per_logical=( - metadata.physical_blocks_per_logical_kv_block - ), - remote_dcp_size=remote_dcp_size, - ), - ) - return self.nixl_wrapper.add_remote_agent(metadata.agent_metadata) - - def initialize_host_xfer_buffer(self, kv_caches: dict[str, torch.Tensor]) -> None: - """Initialize transfer buffer in CPU mem for accelerators - NOT directly supported by NIXL (e.g., tpu) - """ - xfer_buffers: dict[str, torch.Tensor] = {} - try: - for layer_name, kv_cache in kv_caches.items(): - kv_shape = kv_cache.shape - kv_dtype = kv_cache.dtype - permute_shape = False - inv_order = (0, 2, 1, 3) - if not self.use_mla: - assert kv_cache.ndim == 4 - - if self.kv_cache_layout == "LBNHC": - if self.kv_transfer_config.enable_permute_local_kv: - logger.info_once( - "'enable_permute_local_kv' flag is enabled while " - "device KV Layout is LBNHC. Init host buffer with" - " LBHNC to better support Decode/Prefill TP_ratio > 1." - ) - # Since LBNHC will not support Decode/Prefill TP_ratio > 1, - # we can leverage host_buffer for permute. - self.host_buffer_kv_cache_layout = "LBHNC" - else: - # Packed KV layout is logical (B, H, N, 2*D). Allocate - # (B, N, H, 2*D) and view it as logical (B, H, N, 2*D) - # so raw NIXL transfers see NHD physical strides. - kv_shape = tuple(kv_shape[i] for i in inv_order) - permute_shape = True - - xfer_buffers[layer_name] = torch.empty( - kv_shape, dtype=kv_dtype, device="cpu" - ) - if permute_shape: - xfer_buffers[layer_name] = xfer_buffers[layer_name].permute( - inv_order - ) - except MemoryError as e: - logger.error("NIXLConnectorWorker gets %s.", e) - raise - - self.host_xfer_buffers = xfer_buffers - - def set_host_xfer_buffer_ops(self, copy_operation: CopyBlocksOp): - """Assign copy (d2h, h2d) operations when host buffer is used.""" - # Set a no-op if the host buffer is not cpu. - if self.kv_buffer_device != "cpu": - return - # Set a no-op if self.device_type is 'cpu'. - if self.device_type == "cpu": - return - assert self.use_host_buffer - self.copy_blocks = copy_operation - - def _log_failure( - self, - failure_type: str, - req_id: str | None, - msg: str = "", - error: BaseException | None = None, - meta: ReqMeta | None = None, - **extra_context, - ): - """Log transfer failure with structured context for easier debugging.""" - context: dict[str, Any] = { - "failure_type": failure_type, - "request_id": req_id, - "engine_id": self.engine_id, - } - if meta is None and req_id is not None: - # Try to get metadata from in progress transfers when not provided - meta = self._recving_metadata.get(req_id) - - if meta and meta.remote: - context.update( - { - "remote_engine_id": meta.remote.engine_id, - "remote_request_id": meta.remote.request_id, - "remote_host": meta.remote.host, - "remote_port": meta.remote.port, - "num_local_blocks": sum( - len(group) for group in meta.local_block_ids - ), - "num_remote_blocks": sum( - len(group) for group in meta.remote.block_ids - ), - "local_block_ids_sample": meta.local_block_ids[0][:10] - if meta.local_block_ids - else [], - } - ) - - context.update(extra_context) - if msg: - failure_type = f"{failure_type}. {msg}" - - logger.error( - "NIXL transfer failure: %s | Context: %s", - failure_type, - context, - exc_info=error is not None, - stacklevel=2, - ) - - def _ensure_handshake( - self, - engine_id: EngineId, - host: str, - port: int, - tp_size: int, - dcp_size: int = 1, - pp_size: int = 1, - notif_agents_only: bool = False, - ) -> Future[tuple[dict[tuple[int, int], str], float]] | None: - """Ensure a handshake is in-flight (or already done) for *engine_id*. - - Returns the ``Future`` if a handshake is pending (or was just - started), or ``None`` if the handshake already completed - successfully. Callers can attach per-request callbacks to the - returned future. - Failures to handshake are logged and the request is marked as failed. - """ - self._evict_stale_engines() - with self._handshake_lock: - if engine_id in self._remote_agents: - return None - fut = self._handshake_futures.get(engine_id) - if fut is not None: - return fut - fut = self._handshake_initiation_executor.submit( - self._nixl_handshake, - host, - port, - tp_size, - engine_id, - dcp_size, - pp_size, - notif_agents_only, - ) - self._handshake_futures[engine_id] = fut - - def done_callback( - f: Future[tuple[dict[tuple[int, int], str], float]], - eid=engine_id, - ): - with self._handshake_lock: - del self._handshake_futures[eid] - try: - remote_agents, clock_offset = f.result() - self._remote_agents[eid] = remote_agents - self._engine_clock_offset[eid] = clock_offset - self._engine_last_active[eid] = time.perf_counter() - self._track_remote_engine_replacement(eid, host, port) - except Exception as e: - self._log_failure( - failure_type="handshake_setup_failed", - req_id=None, - error=e, - remote_engine_id=eid, - ) - # Count once per handshake, regardless of waiting requests. - self.xfer_stats.record_failed_handshake() - - fut.add_done_callback(done_callback) - return fut - - def _background_nixl_handshake( - self, req_id: str, remote_engine_id: EngineId, meta: ReqMeta - ): - # Do NIXL handshake in background and add to _ready_requests when done. - assert meta.remote is not None - fut = self._ensure_handshake( - remote_engine_id, - meta.remote.host, - meta.remote.port, - meta.tp_size, - meta.dcp_size, - meta.pp_size, - ) - if fut is None: - # Already handshaked — only happens if caller does not pre-check. - self._ready_requests.put((req_id, meta)) - return - - # Check handshake success before proceeding with request. - def request_ready(f: Future[Any], entry=(req_id, meta)): - try: - f.result() - self._ready_requests.put(entry) - except Exception as e: - self._log_failure( - failure_type="handshake_failed", - req_id=req_id, - error=e, - meta=meta, - ) - self._failed_recv_reqs.put(req_id) - - fut.add_done_callback(request_ready) - - def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]): - """Register the KV Cache data in nixl.""" - use_layer_name_routing = self._requires_layer_name_routing() - route_packed_layers = self._has_packed_cache and use_layer_name_routing - self.transfer_topo = TransferTopology( - tp_rank=self.transfer_tp_rank, - tp_size=self.transfer_tp_size, - block_size=self.block_size, - engine_id=self.engine_id, - is_mla=self.use_mla, - total_num_kv_heads=1 - if self.use_mla - else self.model_config.get_total_num_kv_heads(), - attn_backends=self.attn_backends, - dcp_size=self.dcp_size, - # SSM States come in tuples (ssm, conv) - tensor_shape=next(iter(kv_caches.values())).shape - if not self._has_mamba - else None, - is_mamba=self._has_mamba, - ) - backend_name = self.backend_name - if self._supports_pp_hma and self._has_packed_cache: - # PP can reorder the same backends across stages of a packed model. - backend_name = ",".join( - sorted({backend.get_name() for backend in self.attn_backends}) - ) - self.compat_hash = compute_nixl_compatibility_hash( - self.vllm_config, - backend_name, - transfer_mode=self._TRANSFER_MODE, - ) - - if self._is_csa_linear and self.use_host_buffer: - raise NotImplementedError( - "NIXL host staging does not preserve CSA-linear shared tensors." - ) - - if self.use_host_buffer: - self.initialize_host_xfer_buffer(kv_caches=kv_caches) - assert len(self.host_xfer_buffers) == len(kv_caches), ( - f"host_buffer: {len(self.host_xfer_buffers)}, " - f"kv_caches: {len(kv_caches)}" - ) - xfer_buffers = self.host_xfer_buffers - else: - xfer_buffers = kv_caches - assert not self.host_xfer_buffers, ( - "host_xfer_buffer should not be initialized when " - f"kv_buffer_device is {self.kv_buffer_device}" - ) - - logger.info( - "Registering KV_Caches. use_mla: %s, kv_buffer_device: %s, " - "use_host_buffer: %s", - self.use_mla, - self.kv_buffer_device, - self.use_host_buffer, - ) - - registration_ranges: dict[tuple[int, str], tuple[int, int, int]] = {} - region_mem_types: list[str] = [] - seen_base_addresses: list[int] = [] - self._ssm_region_indices = [] - self._scratch_region_indices = [] - self._ple_block_len = None - self._ple_region_index = None - - # A shared packed row may transfer whole only if every page in it is - # replicated across TP ranks: MLA without SSM state. Head-sharded - # attention or SSM state (e.g. Qwen3.8-Flash-Next) must stay split. - packed_storage = ( - _share_storage_and_block_stride(list(xfer_buffers.values())) - and self.use_mla - and not self._has_mamba - ) - if route_packed_layers and any( - not isinstance(spec, (MLAAttentionSpec, SlidingWindowMLASpec)) - for spec in self._layer_specs.values() - ): - raise NotImplementedError("PP push with packed KV caches requires MLA") - - track_region_layers = self._tracks_region_layers() - region_layers: list[list[str]] = [] - packed_member_layouts: dict[str, tuple[int, int]] = {} - - # K and V are packed into the content dim, so each attention layer is a - # single NIXL region whose block transfers as one unit. Mamba layers instead - # register separate conv/ssm sub-regions (see `_build_mamba_local`). - # P and D may allocate equivalent transferable layers in different - # cache-group orders. Keep their region lists aligned without putting - # layers.10 before layers.2, which would break PP region slicing. - layer_names = ( - xfer_buffers - if self._is_csa_linear - else sorted( - xfer_buffers, - key=_region_sort_key, - ) - ) - for layer_name in layer_names: - cache = xfer_buffers[layer_name] - # NOTE (NickLucche) Hybrid SSM mamba/FA physical page_size may differ when - # kernel requires a specific block size. This leads to SSM and FA layers - # having different num_blocks. - # `_physical_blocks_per_logical_kv_block` ratio is used to adjust for this. - layer_spec = self._layer_specs.get(layer_name) - if layer_spec is None: - logger.debug( - "Skipping layer %s as no KVCache spec is present. " - "This is likely because the layer is sharing its KV cache", - layer_name, - ) - continue - group_index = self.kv_cache_config.transfer_group_index_by_layer[layer_name] - # `layer_spec.page_size_bytes` only accounts for logical page_size, that is - # the page_size assuming constant `self._logical_num_blocks`. - physical_page_size = ( - layer_spec.page_size_bytes - if isinstance(layer_spec, MambaSpec) - else layer_spec.page_size_bytes - // self._physical_blocks_per_logical_kv_block - ) - group = self.kv_cache_config.transfer_groups[group_index] - if group.host_resident: - logical_num_blocks = self.kv_cache_config.hisparse_host_num_blocks - assert logical_num_blocks is not None - else: - logical_num_blocks = self.kv_cache_config.num_blocks - group_id = group_index - num_blocks = ( - logical_num_blocks - if isinstance(layer_spec, MambaSpec) - else logical_num_blocks * self._physical_blocks_per_logical_kv_block - ) - is_mla_region = isinstance( - layer_spec, (MLAAttentionSpec, SlidingWindowMLASpec) - ) - logger.debug( - "Registering layer %s with cache shape: %s", layer_name, cache.shape - ) - storage = cache.untyped_storage() - storage_addr = storage.data_ptr() - is_host_resident = group.host_resident - if cache.device.type == "cpu": - mem_type = "DRAM" - region_device_id = 0 - else: - mem_type = self.nixl_memory_type - region_device_id = max(cache.get_device(), 0) - self.device_id = region_device_id - if is_host_resident: - registration_base = cache.data_ptr() - registration_len = cache.nbytes - else: - registration_base = storage_addr - registration_len = storage.nbytes() - storage_key = (registration_base, mem_type) - if storage_key not in registration_ranges: - registration_ranges[storage_key] = ( - registration_base, - registration_base + registration_len, - region_device_id, - ) - - if isinstance(layer_spec, MambaSpec): - physical_ratio = self._physical_blocks_per_logical_kv_block - block_len = physical_page_size // physical_ratio - block_stride = physical_page_size - if self._is_csa_linear and cache.ndim != 1: - # CSA-linear packs multiple cache owners in each block, so - # state views advance by the packed block stride. - block_stride = cache.stride(0) * cache.element_size() - assert block_stride % physical_ratio == 0 - region_specs = [ - (cache.data_ptr(), block_len, block_stride // physical_ratio) - ] - else: - if cache.ndim == 1: - # Flat byte view: HMA tensors shared between layer types carry - # no block dimension, so stride(0) is 1 byte. Blocks abut, so - # the stride is the page size. - block_stride = physical_page_size - else: - block_stride = cache.stride(0) * cache.element_size() - # A connector that registers the backing as one memory region - # may need its length rounded up to a page boundary. Such a tail - # is shorter than one block, so it cannot make the storage - # segmented. - storage_is_block_major = ( - 0 <= registration_len - num_blocks * block_stride < block_stride - ) - # A view with a contiguous page can own a strided region even - # when sibling layers share the same backing allocation. - page_contiguous = ( - cache.ndim > 1 - and cache[0].is_contiguous() - and cache[0].nbytes == physical_page_size - ) - # Block-outermost MLA: one block row holds every layer's page. - packed_mla = ( - self._has_packed_cache - and not self._has_mamba - and packed_storage - and storage_is_block_major - and is_mla_region - ) - if packed_mla and use_layer_name_routing: - # PP>1 prefill stage: register only this layer's page of each row. - region_specs = [ - (cache.data_ptr(), physical_page_size, block_stride) - ] - elif packed_mla and (self.pp_size == 1 or not page_contiguous): - # Register whole rows so a block is a single descriptor. Also, - # advertise where this layer's page sits so a remote with - # LB* can still address it (HiSparse). - storage_block_len = block_stride - region_specs = [ - (registration_base, storage_block_len, storage_block_len) - ] - offset = cache.data_ptr() - storage_addr - assert offset >= 0 and offset + physical_page_size <= block_stride - packed_member_layouts[layer_name] = (offset, physical_page_size) - elif storage_is_block_major and ( - not page_contiguous - and ((packed_storage and is_mla_region) or not self._is_csa_linear) - ): - # TODO(Lucas): handle TP slicing for packed_storage; for now - # restrict to MLA (DSv4) where kv is replicated. - # The complete storage row is one block stride. Dividing the - # registered length instead would fold the padding tail in. - storage_block_len = block_stride - region_specs = [ - (registration_base, storage_block_len, storage_block_len) - ] - elif storage_is_block_major: - region_specs = [ - (cache.data_ptr(), physical_page_size, block_stride) - ] - else: - segment_bytes = num_blocks * block_stride - if cache.nbytes % segment_bytes != 0: - raise AssertionError( - "KV cache view cannot be partitioned into NIXL regions: " - f"layer={layer_name}, cache_nbytes={cache.nbytes}, " - f"num_blocks={num_blocks}, block_stride={block_stride}, " - f"physical_page_size={physical_page_size}, " - f"cache_shape={tuple(cache.shape)}, " - f"cache_stride={tuple(cache.stride())}" - ) - num_segments = cache.nbytes // segment_bytes - region_block_len = ( - block_stride if num_segments > 1 else physical_page_size - ) - region_specs = [ - ( - cache.data_ptr() + segment_idx * segment_bytes, - region_block_len, - block_stride, - ) - for segment_idx in range(num_segments) - ] - - for base_addr, block_len, block_stride in region_specs: - # A packed PP producer keeps one region per layer, so aliases at - # one address (a layer and its SWA view) stay distinct. - if base_addr in seen_base_addresses and not route_packed_layers: - region_index = seen_base_addresses.index(base_addr) - assert region_mem_types[region_index] == mem_type - self._region_is_mla[region_index] |= is_mla_region - if is_mla_region: - self.block_len_per_layer[region_index] = block_len - self.block_stride_per_layer[region_index] = block_stride - self.region_num_blocks[region_index] = num_blocks - if self.region_group_ids[region_index] != group_id: - self.region_group_ids[region_index] = _SHARED_REGION_GROUP_ID - else: - region_index = len(seen_base_addresses) - seen_base_addresses.append(base_addr) - self.block_len_per_layer.append(block_len) - self.block_stride_per_layer.append(block_stride) - self.region_group_ids.append(group_id) - self.region_names.append(layer_name) - self.region_num_blocks.append(num_blocks) - self._region_is_mla.append(is_mla_region) - region_mem_types.append(mem_type) - - if track_region_layers: - if region_index == len(region_layers): - region_layers.append([layer_name]) - else: - region_layers[region_index].append(layer_name) - - if isinstance(layer_spec, MambaSpec): - if layer_spec.tp_replicated: - assert self._is_csa_linear - assert self._ple_region_index in (None, region_index) - self._ple_region_index = region_index - self._ple_block_len = block_len - elif region_index not in self._ssm_region_indices: - self._ssm_region_indices.append(region_index) - elif ( - isinstance(layer_spec, CircularBufferSpec) - and region_index not in self._scratch_region_indices - ): - self._scratch_region_indices.append(region_index) - - # When there's a mismatch between kbs<>bs, we rely on HMA to ensure - # caches are either [NB, PS] or [NB*r, PS/r] where r is bs/kbs. - if ( - self._physical_blocks_per_logical_kv_block == 1 - and cache.shape[0] < num_blocks - ): - raise AssertionError( - "KV cache tensor has too few " - f"blocks; layer={layer_name}, " - f"expected_num_blocks={num_blocks}, " - f"cache_shape={tuple(cache.shape)}, " - f"cache_stride={tuple(cache.stride())}, " - f"layer_spec={type(layer_spec).__name__}, " - f"backend={self.backend_name}, " - "all_backends=" - f"{[backend.get_name() for backend in self.attn_backends]}, " - f"kv_cache_layout={self.kv_cache_layout}" - ) - - logger.debug( - "Different block lengths collected: %s", set(self.block_len_per_layer) - ) - assert ( - len(self.block_len_per_layer) - == len(seen_base_addresses) - == len(self._region_is_mla) - == len(self.block_stride_per_layer) - == len(self.region_group_ids) - == len(self.region_names) - == len(self.region_num_blocks) - ) - # Descriptor ids must be region-ordered, matching the remote side. - self._scratch_region_indices.sort() - - self.kv_caches_base_addr[self.engine_id][self.tp_rank] = seen_base_addresses - self.num_regions = len(seen_base_addresses) - self.region_mem_types = region_mem_types - self._uses_region_group_mapping = len(set(self.region_group_ids)) > 1 - self._mixed_mem_types = len(set(region_mem_types)) > 1 - if self._has_mamba: - self.region_num_blocks = [self.num_blocks] * self.num_regions - - self._set_region_layers(region_layers) - self._packed_member_layouts = packed_member_layouts - - if self.pp_size > 1 and not use_layer_name_routing: - start_layer, end_layer = self.model_config.get_layers_start_end_indices( - self.vllm_config.parallel_config - ) - num_local_layers = end_layer - start_layer - assert num_local_layers > 0 and self.num_regions % num_local_layers == 0 - regions_per_layer = self.num_regions // num_local_layers - self._remote_region_offset = regions_per_layer * start_layer - - # Total local FA descriptors (boundary between FA and mamba descs). - xfer_region_num_blocks = ( - [self.region_num_blocks[i] for i in self._transfer_layer_region_indices] - if self._transfer_layer_region_indices - else self.region_num_blocks - ) - self.num_descs = sum(xfer_region_num_blocks) - - self._mixed_mem_types = len(set(region_mem_types)) > 1 - self._skip_dram_xfer = self._leaves_dram_xfer_to_rank0() - if self._mixed_mem_types: - assert self.use_mla and not self._has_mamba, ( - "Mixed-device KV registration is only supported for MLA " - "models without Mamba layers." - ) - for mem_type in sorted(set(region_mem_types)): - if mem_type == "DRAM" and self._skip_dram_xfer: - continue - ranges_for_mem_type = [ - (start, end - start, device_id, "") - for (_, cache_mem_type), ( - start, - end, - device_id, - ) in registration_ranges.items() - if cache_mem_type == mem_type - ] - descs = self.nixl_wrapper.get_reg_descs(ranges_for_mem_type, mem_type) - self.nixl_wrapper.register_memory(descs, backends=self.nixl_backends) - self._registered_descs.append(descs) - - self.device_kv_caches = kv_caches - self.dst_num_blocks[self.engine_id] = self.num_blocks - self.dst_region_num_blocks[self.engine_id] = xfer_region_num_blocks - self.dst_region_group_ids[self.engine_id] = self.region_group_ids - self.dst_uses_region_group_mapping[self.engine_id] = ( - self._uses_region_group_mapping - ) - self.dst_region_mem_types[self.engine_id] = self.region_mem_types - - if self._has_mamba: - logger.info( - "Hybrid SSM registration: num_blocks=%s, " - "logical_num_blocks=%s, ratio=%s, num_regions=%s, " - "num_descs=%s, mamba_ssm_size=%s, block_len_per_layer=%s", - self.num_blocks, - self._logical_num_blocks, - self._physical_blocks_per_logical_kv_block, - self.num_regions, - self.num_descs, - self._mamba_ssm_size, - set(self.block_len_per_layer), - ) - - # Register local/src descr for NIXL xfer. - self.src_xfer_handles_by_block_size[self.block_size], self.src_blocks_data = ( - self.register_local_xfer_handler(self.block_size) - ) - - # After KV Caches registered, listen for new connections. - agent_metadata = NixlAgentMetadata( - engine_id=self.engine_id, - agent_metadata=self.nixl_wrapper.get_agent_metadata(), - device_id=self.device_id, - kv_caches_base_addr=self.kv_caches_base_addr[self.engine_id][self.tp_rank], - num_blocks=self.num_blocks, - block_lens=self.block_len_per_layer, - block_strides=self.block_stride_per_layer, - kv_cache_layout=self.kv_cache_layout - if not self.use_host_buffer - else self.host_buffer_kv_cache_layout, - block_size=self.block_size, - ssm_sizes=self._mamba_ssm_size, - ple_block_len=self._ple_block_len, - attn_backend_name=self.backend_name, - physical_blocks_per_logical_kv_block=( - self._physical_blocks_per_logical_kv_block - ), - region_num_blocks=self.region_num_blocks, - region_group_ids=self.region_group_ids, - region_names=self.region_names, - region_mem_types=self.region_mem_types, - dcp_size=self.dcp_size, - pcp_size=self.pcp_size, - region_members=self.region_members, - packed_member_layouts=self._packed_member_layouts, - ) - # Wrap metadata in payload with hash for defensive decoding - assert self.compat_hash is not None - encoder = msgspec.msgpack.Encoder() - self.xfer_handshake_metadata = NixlHandshakePayload( - compatibility_hash=self.compat_hash, - agent_metadata_bytes=encoder.encode(agent_metadata), - ) - - def _ple_page_len(self) -> int: - """Transfer length of one PLE state page. - - The PLE page shares its region with the first page of other cache - groups (block-outer layout), whose block_len may be smaller; the - descriptors must cover the whole PLE page or part of the conv state - never reaches the decode side.""" - assert self._ple_region_index is not None - assert self._ple_block_len is not None, ( - "PLE region registered without its page length" - ) - return self._ple_block_len - - def _build_mamba_local(self, base_addresses: list[int]) -> np.ndarray: - """Build desc regions (conv sub-projections + ssm) per layer for - local mamba blocks with DS conv layout, as Nx5 uint64 strided descs. - - A Mamba block interleaves conv and SSM state, which crucially differ in - size, so the two are indexed as separate sub-regions. Attention blocks - instead pack K and V into the content dim and transfer as a single unit. - Reference diagram: - KVCacheTensor (Shared) - / \\ - / \\ - / \\ - Attention (FlashInfer) View Mamba View - | | - | | - +-------------------+ +-------------------+ - | KVCacheTensor | | KVCacheTensor | - | | | | - |<----- page ------>| |<----- page ------->| - | size | | size | - | Key 0 | Val 0 | |Conv 0 | SSM 0 | - | Key 1 | Val 1 | |Conv 1 | SSM 1 | - | ... | ... | | ... | ... | - | Key N-2 | Val N-2 | |Conv N-2| SSM N-2 | - | Key N-1 | Val N-1 | |Conv N-1| SSM N-1 | - +-------------------+ +--------------------+ - |1st_split-2nd_split| |1st_split-2nd_split | - - Mamba state blocks are indivisible (not token-extent data), so the - descriptors always use the local page geometry regardless of any - attention block-size ratio; their desc ids are likewise never - ratio-expanded (see _compute_desc_ids). - """ - assert base_addresses, "Local KV cache base addresses must not be empty." - assert self._conv_decomp is not None - conv_offsets = self._conv_decomp.local_conv_offsets - conv_size, ssm_size = self._mamba_ssm_size - num_blocks = self._logical_num_blocks - physical_per_logical = self._physical_blocks_per_logical_kv_block - device_id = self.device_id - strided_descs: list[tuple[int, int, int, int, int]] = [] - - region_indices = self._ssm_region_indices or range(len(base_addresses)) - for i in region_indices: - base_addr = base_addresses[i] - block_stride = self.block_stride_per_layer[i] * physical_per_logical - for off, sz in conv_offsets: - strided_descs.append( - (base_addr + off, sz, device_id, block_stride, num_blocks) - ) - # SSM temporal state follows the conv state. - strided_descs.append( - (base_addr + conv_size, ssm_size, device_id, block_stride, num_blocks) - ) - - if (region_index := self._ple_region_index) is not None: - block_len = self._ple_page_len() * physical_per_logical - block_stride = ( - self.block_stride_per_layer[region_index] * physical_per_logical - ) - base_addr = base_addresses[region_index] - strided_descs.append( - (base_addr, block_len, device_id, block_stride, num_blocks) - ) - - return self._pack_strided_descs(strided_descs) - - def _build_mamba_remote( - self, - nixl_agent_meta: NixlAgentMetadata, - tp_ratio: int, - transfer_info: EngineTransferInfo, - ) -> np.ndarray: - """Build remote desc regions (conv sub-projections + ssm) per layer. - For hetero-TP, each D rank reads only its sub-projection slice from - the P rank. Returns Nx5 uint64 strided descs.""" - assert nixl_agent_meta.kv_caches_base_addr, ( - "Remote KV cache base addresses must not be empty." - ) - assert self._conv_decomp is not None - effective_ratio = max(tp_ratio, 1) - # Mamba conv state is always TP-sharded, even when attention KV - # is replicated (num_kv_heads < tp_size). - local_offset = self.tp_rank % effective_ratio - conv_size_remote = nixl_agent_meta.ssm_sizes[0] - - conv_offsets = self._conv_decomp.remote_conv_offsets(local_offset, tp_ratio) - if tp_ratio >= 1: - ssm_read_size = self._mamba_ssm_size[1] - else: - ssm_read_size = nixl_agent_meta.ssm_sizes[1] - - remote_physical_per_logical = transfer_info.remote_physical_blocks_per_logical - num_blocks = nixl_agent_meta.num_blocks // remote_physical_per_logical - device_id = nixl_agent_meta.device_id - - strided_descs: list[tuple[int, int, int, int, int]] = [] - # NOTE (ZhanqiuHu): use per-layer block_lens[i], not [0], in case - # block lengths vary across layers (e.g. MLA). - region_indices = self._ssm_region_indices or range( - len(nixl_agent_meta.kv_caches_base_addr) - ) - for i in region_indices: - base_addr = nixl_agent_meta.kv_caches_base_addr[i] - block_stride = ( - nixl_agent_meta.block_strides[i] * remote_physical_per_logical - ) - for off, sz in conv_offsets: - strided_descs.append( - (base_addr + off, sz, device_id, block_stride, num_blocks) - ) - # SSM temporal state is also TP-sharded on the heads dimension. - ssm_addr = base_addr + conv_size_remote + local_offset * ssm_read_size - strided_descs.append( - (ssm_addr, ssm_read_size, device_id, block_stride, num_blocks) - ) - - if (region_index := self._ple_region_index) is not None: - local_block_len = ( - self._ple_page_len() * self._physical_blocks_per_logical_kv_block - ) - # Same NIXL_CONNECTOR_VERSION on both sides, so the field is set. - assert nixl_agent_meta.ple_block_len is not None - remote_block_len = ( - nixl_agent_meta.ple_block_len * remote_physical_per_logical - ) - if local_block_len != remote_block_len: - raise ValueError( - "PLE pages require identical P/D geometry: " - f"local={local_block_len}, remote={remote_block_len}." - ) - remote_block_stride = ( - nixl_agent_meta.block_strides[region_index] - * remote_physical_per_logical - ) - addr = nixl_agent_meta.kv_caches_base_addr[region_index] - strided_descs.append( - (addr, remote_block_len, device_id, remote_block_stride, num_blocks) - ) - - return self._pack_strided_descs(strided_descs) - - @staticmethod - def _pack_strided_descs( - strided_descs: list[tuple[int, int, int, int, int]], - ) -> np.ndarray: - """Pack (addr, len, dev_id, stride, count) rows into an Nx5 uint64 array. - - Each row describes `count` blocks of `len` bytes whose starts are - `stride` bytes apart; NIXL >= 1.5 prepares this layout directly. - """ - return np.array(strided_descs, dtype=np.uint64).reshape(-1, 5) - - @staticmethod - def _block_strided_descs( - base_addr: int, - block_len: int, - block_stride: int, - num_blocks: int, - block_size_ratio: int, - device_id: int, - ) -> np.ndarray: - """Nx5 strided descs for `num_blocks` blocks, each split into - `block_size_ratio` sub-blocks of `block_len` bytes. - - Evenly spaced sub-blocks collapse into one strided desc. In a padded - block the sub-blocks abut only within the block, so each block gets - its own. - """ - if block_size_ratio == 1 or block_stride == block_size_ratio * block_len: - stride = block_stride // block_size_ratio - count = num_blocks * block_size_ratio - return np.array( - [[base_addr, block_len, device_id, stride, count]], dtype=np.uint64 - ) - strided_descs = np.empty((num_blocks, 5), dtype=np.uint64) - strided_descs[:, 0] = ( - base_addr + np.arange(num_blocks, dtype=np.uint64) * block_stride - ) - strided_descs[:, 1] = block_len - strided_descs[:, 2] = device_id - strided_descs[:, 3] = block_len - strided_descs[:, 4] = block_size_ratio - return strided_descs - - @staticmethod - def _strided_desc_starts(strided_descs: np.ndarray) -> np.ndarray: - """Flat block index of each strided desc's first block.""" - counts = strided_descs[:, 4] - return np.cumsum(counts) - counts - - def _build_fa_local( - self, - base_addresses: list[int], - block_size_ratio: int, - ) -> np.ndarray: - """Build local FA descriptors as Nx5 uint64 strided descs.""" - assert self.transfer_topo is not None - assert base_addresses, "Local KV cache base addresses must not be empty." - device_id = self.device_id - region_indices = self._transfer_layer_region_indices or range( - len(base_addresses) - ) - parts = [ - self._block_strided_descs( - base_addresses[i], - self.block_len_per_layer[i] // block_size_ratio, - self.block_stride_per_layer[i], - self.region_num_blocks[i], - block_size_ratio, - device_id, - ) - for i in region_indices - ] - return np.concatenate(parts) - - def _build_fa_remote( - self, - plan: TPMapping, - nixl_agent_meta: NixlAgentMetadata, - block_size_ratio: int, - ) -> np.ndarray: - """Build remote FA descriptors as Nx5 uint64 strided descs.""" - assert self.transfer_topo is not None - assert nixl_agent_meta.kv_caches_base_addr, ( - "Remote KV cache base addresses must not be empty." - ) - fa_group_idx = next( - i for i, t in enumerate(self._group_spec_types) if _is_attention_spec(t) - ) - # SPLIT regions read their head slice from this many remote ranks at a - # per-rank offset; REPLICATE regions read the whole block once. - split_reads = len(plan.source_ranks_per_group[fa_group_idx]) - region_num_blocks = nixl_agent_meta.region_num_blocks or [ - nixl_agent_meta.num_blocks - ] * len(nixl_agent_meta.kv_caches_base_addr) - device_id = nixl_agent_meta.device_id - block_strides = nixl_agent_meta.block_strides - strided_descs: list[tuple[int, int, int, int, int]] = [] - use_layer_name_routing = bool(self._transfer_layer_region_indices) - for i, base_addr in enumerate(nixl_agent_meta.kv_caches_base_addr): - local_region = ( - self._transfer_layer_region_indices[i] if use_layer_name_routing else i - ) - replicated = self._is_region_replicated(local_region) - # Read our whole local region size from remote.. - local_block_len = self.block_len_per_layer[local_region] - remote_kv_block_len = local_block_len // block_size_ratio - if block_size_ratio > 1: - # ..using remote kv_block_len as transfer unit - local_block_len = remote_kv_block_len - - # REPLICATE reads the whole block once at offset 0; SPLIT gathers - # its head slice from `split_reads` remote ranks at a per-rank offset. - num_reads = 1 if replicated else split_reads - rank_offset = ( - 0 if replicated else plan.rank_offset_factor * remote_kv_block_len - ) - local_block_len = local_block_len // num_reads - - addr = base_addr + rank_offset - count = region_num_blocks[i] - strided_descs.append( - (addr, local_block_len, device_id, block_strides[i], count) - ) - return self._pack_strided_descs(strided_descs) - - def register_local_xfer_handler( - self, - block_size: int, - ) -> tuple[int, np.ndarray]: - """Function used for register local xfer handler with local block_size or - Remote block_size. - - When local block_size is same as remote block_size, we use local block_size - to register local_xfer_handler during init. - - When remote block size is less than local block size, we need to use - register another local_xfer_handler using remote block len to ensure - data copy correctness. - """ - assert self.transfer_topo is not None - block_size_ratio = self.block_size // block_size - local_base_addresses = self.kv_caches_base_addr[self.engine_id][self.tp_rank] - - blocks_data = self._build_fa_local(local_base_addresses, block_size_ratio) - logger.debug( - "Created %s blocks for src engine %s and rank %s on device id %s", - blocks_data[:, 4].sum(), - self.engine_id, - self.tp_rank, - self.device_id, - ) - if self._has_mamba: - assert self.num_descs * block_size_ratio == blocks_data[:, 4].sum() - # TODO (ZhanqiuHu): For homogeneous TP (tp_ratio == 1), the 3-descs split - # is unnecessary — a single conv desc per block suffices. Consider - # adding a fast path that falls back to the standard 2-region - # registration (_build_fa_local mamba=True) when no hetero-TP - # remote has been seen. Currently we always register 4 regions - # because local descs are created before knowing the remote TP. - logger.debug("Registering local Mamba descriptors (4 regions/layer)") - mamba = self._build_mamba_local(local_base_addresses) - blocks_data = np.concatenate([blocks_data, mamba]) - - if self._mixed_mem_types: - desc_is_dram = np.concatenate( - [ - np.full( - count * block_size_ratio, - mem_type == "DRAM", - dtype=bool, - ) - for mem_type, count in zip( - self.region_mem_types, - self.region_num_blocks, - strict=True, - ) - ] - ) - assert len(desc_is_dram) == blocks_data[:, 4].sum() - desc_pos = np.empty(len(desc_is_dram), dtype=np.int64) - dram_idx = np.where(desc_is_dram)[0] - vram_idx = np.where(~desc_is_dram)[0] - desc_pos[dram_idx] = np.arange(len(dram_idx), dtype=np.int64) - desc_pos[vram_idx] = np.arange(len(vram_idx), dtype=np.int64) - self._desc_is_dram_by_block_size[block_size] = desc_is_dram - self._desc_pos_by_block_size[block_size] = desc_pos - # DRAM descriptors are registered under CPU device 0. - # A strided desc never spans regions, so its first block decides. - strided_is_dram = desc_is_dram[self._strided_desc_starts(blocks_data)] - blocks_data[strided_is_dram, 2] = 0 - if self._skip_dram_xfer: - self._dram_src_handles_by_block_size[block_size] = None - else: - self._dram_src_handles_by_block_size[block_size] = ( - self.nixl_wrapper.prep_xfer_dlist( - "NIXL_INIT_AGENT", - blocks_data[strided_is_dram], - mem_type="DRAM", - ) - ) - device_handle = self.nixl_wrapper.prep_xfer_dlist( - "NIXL_INIT_AGENT", - blocks_data[~strided_is_dram], - mem_type=self.nixl_memory_type, - ) - return device_handle, blocks_data - - # NIXL_INIT_AGENT to be used for preparations of local descs. - handle = self.nixl_wrapper.prep_xfer_dlist( - "NIXL_INIT_AGENT", blocks_data, mem_type=self.region_mem_types[0] - ) - return handle, blocks_data - - def add_remote_agent( - self, - nixl_agent_meta: NixlAgentMetadata, - remote_tp_rank: int = 0, - remote_tp_size: int = 1, - remote_dcp_size: int = 1, - ) -> str: - """Add the remote NIXL agent and prepare the descriptors for reading cache - blocks from remote. - - In particular, handle both homogeneous and heterogeneous TP. The former - requires local rank_i to read from remote rank_i. - The latter, in the case of D.world_size < P.world_size, requires that a - local (D) TP worker reads from multiple remote (P) TP workers. - Conversely, assuming D.world_size > P.world_size, two or more local TP - workers will read from a single remote TP worker. - - Here's an example for the last case described above (non-MLA): - - rank_offset p_remote_tp_rank - (kv split no) - -------------------------------- - 0 0 Worker0 ---- 1st half of KV ----> Worker0 [ KV Cache ] - / - 1 0 Worker1 ---- 2nd half of KV -----/ - - 0 1 Worker2 ---- 1st half of KV ----> Worker1 [ KV Cache ] - / - 1 1 Worker3 ---- 2nd half of KV -----/ - - - Decoder TP workers Prefix TP workers - (world_size=4) (world_size=2) - tp_ratio = 4 // 2 = 2 - - Considering the KV Caches, if P-Worker_i has cache size [2, num_blocksP, kv_heads, block_size, head_dim] - then D-Worker_j has [2, num_blocksD, kv_heads//tp_ratio, block_size, head_dim]. Mind the "LBHNC" layout format. - Assuming num_blocksD >= num_blocksP, D-Worker0 reads from P-Worker0 by preparing the kv_heads//tp_ratio - first heads from all the slots of all the blocks. D-Worker1 will do the same, but reading the second split - along the kv_heads dimension, and so forth until "tp_ratio" D TP workers have pulled from P-Worker0. - - Note that the above will also hold true for the homogeneous TP case, where tp_ratio evaluates to 1. - - Regarding MLA case, the cache is replicated across TP workers so the rank_offset will just always be 0 - so that the whole cache is shared by "tp_ratio" D TP workers. - - For Mamba hetero-TP, both tp_ratio > 0 (D_TP > P_TP) and - tp_ratio < 0 (P_TP > D_TP) are supported by the 3-read transfer. - """ # noqa: E501 - engine_id = nixl_agent_meta.engine_id - # TODO re-evaluate refreshing for scaling/recovery - if (0, remote_tp_rank) in self._remote_agents.get(engine_id, {}): - logger.debug( - "Remote agent with engine_id %s and rank" - "%s already exchanged metadata, skip handshake.", - engine_id, - remote_tp_rank, - ) - return self._remote_agents[engine_id][(0, remote_tp_rank)] - - assert self.transfer_topo is not None - transfer_topo = self.transfer_topo - # Number of physical regions registered locally (one per layer/tensor). - num_local_regions = len(self.block_len_per_layer) - if self._transfer_layer_region_indices: - if self.block_size != nixl_agent_meta.block_size: - raise NotImplementedError( - "Attention-HMA push requires identical P/D block sizes." - ) - if remote_tp_size > transfer_topo.tp_size and not self.use_mla: - raise NotImplementedError( - "Attention-HMA push does not support decode TP greater than " - "prefill TP yet" - ) - self._align_remote_regions_by_layer(nixl_agent_meta) - elif nixl_agent_meta.packed_member_layouts and not self._packed_member_layouts: - # With HiSparse, P may be on BL* while D is LB*, as only indexer is on HBM. - # TODO (NickLucche) we may consider not allowing this case at all (P BL*) - # and just clear this function. - self._split_remote_packed_rows(nixl_agent_meta) - elif ( - self.pp_size > 1 - and len(nixl_agent_meta.kv_caches_base_addr) > num_local_regions - ): - # This worker holds a PP layer-slice; the PP=1 remote registered - # the full model. Slice its regions to our layer window so the - # logic below sees congruent local/remote lists. - start = self._remote_region_offset - end = start + num_local_regions - assert len(nixl_agent_meta.kv_caches_base_addr) >= end - nixl_agent_meta.kv_caches_base_addr = nixl_agent_meta.kv_caches_base_addr[ - start:end - ] - nixl_agent_meta.block_lens = nixl_agent_meta.block_lens[start:end] - nixl_agent_meta.block_strides = nixl_agent_meta.block_strides[start:end] - if nixl_agent_meta.region_num_blocks is not None: - nixl_agent_meta.region_num_blocks = nixl_agent_meta.region_num_blocks[ - start:end - ] - if nixl_agent_meta.region_group_ids is not None: - nixl_agent_meta.region_group_ids = nixl_agent_meta.region_group_ids[ - start:end - ] - if nixl_agent_meta.region_names is not None: - nixl_agent_meta.region_names = nixl_agent_meta.region_names[start:end] - if nixl_agent_meta.region_mem_types is not None: - nixl_agent_meta.region_mem_types = nixl_agent_meta.region_mem_types[ - start:end - ] - - if ( - self._packed_member_layouts - and not self._transfer_layer_region_indices - and self._packed_member_layouts != nixl_agent_meta.packed_member_layouts - ): - # Whole rows transfer as one unit, so both sides must pack them alike. - raise RuntimeError( - "Packed KV block rows differ between P and D. Use the same " - "VLLM_KV_CACHE_LAYOUT and block size on both sides." - ) - - ### Register remote engine in TransferTopology (idempotent). - physical_blocks_per_logical = ( - nixl_agent_meta.physical_blocks_per_logical_kv_block - ) - transfer_info = EngineTransferInfo( - remote_tp_size=remote_tp_size, - remote_block_size=nixl_agent_meta.block_size, - remote_block_len=nixl_agent_meta.block_lens[0], - remote_physical_blocks_per_logical=physical_blocks_per_logical, - remote_dcp_size=remote_dcp_size, - ) - transfer_topo.register_remote_engine(engine_id, transfer_info) - logger.info("Transfer plan: %s", transfer_topo.describe(engine_id)) - - self.tp_mappings[engine_id] = compute_tp_mapping( - transfer_topology=transfer_topo, - remote_tp_size=remote_tp_size, - group_spec_types=self._group_spec_types, - remote_dcp_size=remote_dcp_size, - ) - - remote_agent_name = self.nixl_wrapper.add_remote_agent( - nixl_agent_meta.agent_metadata - ) - - # Create dst descs and xfer side handles. TP workers have same #blocks - # so we only register once per engine_id. - # Example: - # block_size_ratio > 1: - # remote: | 0| 1| 2| 3| 4| 5| 6| 7| 8| 9|10|11|12| - # local origin:| 0| 1| 8| 12| - # local mapped:| 0| 1| 2| 3| 4| 5| 6| 7| 8| 9|10|11|12|13|14|15| - block_size_ratio = transfer_topo.block_size_ratio(nixl_agent_meta.block_size) - - if engine_id not in self.dst_num_blocks: - num_remote_regions = len(nixl_agent_meta.kv_caches_base_addr) - self.dst_num_blocks[engine_id] = nixl_agent_meta.num_blocks - self.dst_region_num_blocks[engine_id] = ( - nixl_agent_meta.region_num_blocks - or [nixl_agent_meta.num_blocks] * num_remote_regions - ) - remote_region_group_ids = nixl_agent_meta.region_group_ids or ( - self.region_group_ids - if len(self.region_group_ids) == num_remote_regions - else [0] * num_remote_regions - ) - self.dst_region_group_ids[engine_id] = remote_region_group_ids - self.dst_uses_region_group_mapping[engine_id] = ( - len(set(self.dst_region_group_ids[engine_id])) > 1 - ) - self.dst_region_mem_types[engine_id] = ( - nixl_agent_meta.region_mem_types - or [self.nixl_memory_type] * num_remote_regions - ) - - # Keep track of remote agent kv caches base addresses. - self.kv_caches_base_addr[engine_id][remote_tp_rank] = ( - nixl_agent_meta.kv_caches_base_addr - ) - self._validate_remote_agent_handshake( - nixl_agent_meta, remote_tp_size, remote_dcp_size - ) - - # This is 1 when P and D `--tensor-parallel-size` match. Otherwise, - # this is the ratio between the two sizes. - tp_ratio = transfer_topo.tp_ratio(remote_tp_size) - - logger.debug( - "Registering remote agent (%s, rank %s) memory regions with tp_ratio %s", - engine_id, - remote_tp_rank, - tp_ratio, - ) - - plan = self.tp_mappings[engine_id] - - ### (Optional) Register a local handler at the remote engine's block - ### granularity (remote/prefill blocks smaller than local). - remote_block_size = nixl_agent_meta.block_size - src_blocks_data = self.src_blocks_data - if block_size_ratio > 1: - if remote_block_size not in self.src_xfer_handles_by_block_size: - handle, blocks_data = self.register_local_xfer_handler( - remote_block_size - ) - self.src_xfer_handles_by_block_size[remote_block_size] = handle - self.src_blocks_data_by_block_size[remote_block_size] = blocks_data - src_blocks_data = self.src_blocks_data_by_block_size[remote_block_size] - - ### (Optional) Register local agent memory regions. MLA is not split. - split_key = (tp_ratio, remote_block_size) - if self._needs_split_local_xfer_handles(tp_ratio, plan) and ( - split_key not in self.src_xfer_handles_by_tp_ratio - ): - # Remote tp_size > local tp_size: read from multiple remote ranks. - # Logically "split" own regions into per-source chunks. Hybrid - # MLA+SSM also needs this path: MLA is replicated and read once, - # while the SSM state is sharded across every remote TP rank. - # We only do this once per remote (tp_size, block_size). - self.src_xfer_handles_by_tp_ratio[split_key] = [] - if self._mixed_mem_types: - self._dram_src_handles_by_tp_ratio[split_key] = [] - - for handle_data in self._build_local_splits_from_plan( - plan, - src_blocks_data, - self.num_descs * block_size_ratio, - block_size_ratio, - ): - if self._mixed_mem_types: - desc_is_dram = self._desc_is_dram_by_block_size[remote_block_size] - strided_is_dram = desc_is_dram[ - self._strided_desc_starts(handle_data) - ] - if self._skip_dram_xfer: - dram_handle = None - else: - dram_handle = self.nixl_wrapper.prep_xfer_dlist( - "NIXL_INIT_AGENT", - handle_data[strided_is_dram], - mem_type="DRAM", - ) - self._dram_src_handles_by_tp_ratio[split_key].append(dram_handle) - handle_data = handle_data[~strided_is_dram] - handle = self.nixl_wrapper.prep_xfer_dlist( - "NIXL_INIT_AGENT", handle_data, mem_type=self.nixl_memory_type - ) - self.src_xfer_handles_by_tp_ratio[split_key].append(handle) - - ### Register remote agent memory regions - # With homogeneous TP, D pulls the whole kv cache from corresponding rank. With - # heterogeneous TP, prepare the descriptors by splitting the P KV cache along - # kv_head dim, of D worker's kv_head size (D>P). - # Eg. PTP1 DTP2 => P0 KV:[block0-KV_0 | block0-KV_1..]. - - # Register all remote blocks, but only the corresponding kv heads. - blocks_data = self._build_fa_remote( - plan, - nixl_agent_meta, - block_size_ratio, - ) - logger.debug( - "Created %s blocks for dst engine %s with remote rank %s and local rank %s", - blocks_data[:, 4].sum(), - engine_id, - remote_tp_rank, - self.tp_rank, - ) - if self._has_mamba: - logger.debug( - "Registering remote Mamba blocks for engine %s rank %s", - engine_id, - remote_tp_rank, - ) - mamba = self._build_mamba_remote(nixl_agent_meta, tp_ratio, transfer_info) - blocks_data = np.concatenate([blocks_data, mamba]) - - remote_mem_types = set(self.dst_region_mem_types[engine_id]) - if len(remote_mem_types) != 1: - raise NotImplementedError( - "NIXL pull does not yet support a mixed-memory producer" - ) - self.dst_xfer_side_handles[engine_id][remote_tp_rank] = ( - self.nixl_wrapper.prep_xfer_dlist( - remote_agent_name, blocks_data, mem_type=remote_mem_types.pop() - ) - ) - - return remote_agent_name - - def _validate_remote_agent_handshake( - self, - nixl_agent_meta: NixlAgentMetadata, - remote_tp_size: int, - remote_dcp_size: int = 1, - ): - """Validate the remote agent handshake metadata ensuring the - invariants hold true. - """ - remote_engine_id = nixl_agent_meta.engine_id - - assert self.transfer_topo is not None - remote_info = self.transfer_topo.get_engine_info(remote_engine_id) - assert remote_info.remote_tp_size == remote_tp_size - assert remote_info.remote_dcp_size == remote_dcp_size - # DCP sizes must divide one another; this is what keeps the - # read-slicing math in pull_worker a closed form. - assert ( - self.dcp_size % remote_dcp_size == 0 or remote_dcp_size % self.dcp_size == 0 - ), ( - f"DCP sizes must divide one another: local={self.dcp_size}, " - f"remote={remote_dcp_size} (engine {remote_engine_id})." - ) - if self._has_mamba and self.dcp_size != remote_dcp_size: - raise RuntimeError( - "Hybrid MLA+Mamba NIXL transfers require matching DCP sizes, " - f"got local={self.dcp_size}, remote={remote_dcp_size}." - ) - - tp_ratio = self.transfer_topo.tp_ratio(remote_tp_size) - block_size_ratio = self.transfer_topo.block_size_ratio( - nixl_agent_meta.block_size - ) - # num_kv_heads > tp_size with P_TP > D_TP not supported for non-mamba. - # Mamba models can have replicated FA KV with tp_ratio < 0. - # MLA models do not need to handle kv replication. - if not self.use_mla and not self._has_mamba: - assert not ( - tp_ratio < 0 and self.transfer_topo.is_kv_replicated(remote_engine_id) - ) - - remote_physical_per_logical = ( - nixl_agent_meta.physical_blocks_per_logical_kv_block - ) - if ( - (self.dcp_size > 1 or remote_dcp_size > 1) - and self.region_group_ids - and self.region_group_ids != self.dst_region_group_ids[remote_engine_id] - and ( - self._physical_blocks_per_logical_kv_block != 1 - or remote_physical_per_logical != 1 - ) - ): - raise NotImplementedError( - "DCP region pulls require matching logical and physical block sizes" - ) - if ( - self._has_mamba - and remote_physical_per_logical - != self._physical_blocks_per_logical_kv_block - and self.vllm_config.cache_config.enable_prefix_caching - ): - raise RuntimeError( - "Prefix caching with heterogeneous physical_blocks_per_logical " - "is not supported for Mamba hybrid models. " - f"Local: {self._physical_blocks_per_logical_kv_block}, " - f"Remote: {remote_physical_per_logical}. " - "Disable prefix caching with --disable-prefix-caching." - ) - - if block_size_ratio != 1: - # Heterogeneous block sizes transfer at remote-block granularity; - # the untransferred tail of the last local attention block is - # zeroed in the receive post-process, and mamba state pages - # transfer 1:1 (never sub-split). - assert not self.use_host_buffer, ( - "Heterogeneous block sizes are not supported with host buffer" - ) - kv_cache_layout = ( - self.kv_cache_layout - if not self.use_host_buffer - else self.host_buffer_kv_cache_layout - ) - if not self.use_mla and nixl_agent_meta.kv_cache_layout != kv_cache_layout: - if ( - self.kv_transfer_config.enable_permute_local_kv - and nixl_agent_meta.kv_cache_layout == "LBHNC" - and kv_cache_layout == "LBNHC" - ): - logger.info( - "Remote is LBHNC and local is LBNHC, enabled additional permute " - "on local device KV." - ) - assert not self._is_hma_required, ( - "HMA does not support block size post processing" - ) - self.enable_permute_local_kv = True - else: - raise RuntimeError( - "Heterogeneous TP expects same kv_cache_layout (local " - f"{kv_cache_layout}, remote {nixl_agent_meta.kv_cache_layout}). " - "Or enable experimental feature to receive LBHNC into a local " - "LBNHC cache by setting 'enable_permute_local_kv'=True " - "in --kv-transfer-config." - ) - # if remote_agent used attn is not same as local, - # hint heterogenuous attn post process - if ( - nixl_agent_meta.attn_backend_name != self.backend_name - and self.backend_name in ["CPU_ATTN"] - ): - if self._is_hma_required: - raise RuntimeError( - "heterogeneous attn post process is not supported with HMA" - ) - logger.info( - "[Experimental] CPU_ATTN backend is used, " - "hint heterogeneous attn post process" - ) - self.enable_heterogeneous_attn_post_process = True - - # Heterogeneous TP requires head-splitting, which only works with - # block-contiguous layouts (e.g. LBHNC). MLA and replicated-KV cases - # don't split on heads. Mamba doesn't support heterogeneous TP. - if ( - abs(tp_ratio) != 1 - and not self.use_mla - and not self.transfer_topo.is_kv_replicated(remote_engine_id) - and not KVCacheLayout[kv_cache_layout].is_block_contiguous - and not self.enable_permute_local_kv - ): - raise RuntimeError( - "Heterogeneous TP head-dimension splitting requires contiguous heads. " - "Use a block-contiguous layout (e.g. LBHNC) on the prefill side." - ) - # NVFP4 pages store all heads' data before all heads' scales, so a - # head's bytes are not a contiguous slice of the page. - if ( - not self.use_mla - and self.transfer_topo.local_physical_heads - != max(1, self.transfer_topo.total_num_kv_heads // remote_tp_size) - and any( - isinstance(spec, AttentionSpec) and spec.kv_quant_mode.is_nvfp4 - for spec in self._layer_specs.values() - ) - ): - raise RuntimeError( - "NIXL KV-head re-sharding is not supported for NVFP4 KV cache " - f"(local TP {self.transfer_topo.tp_size}, remote TP " - f"{remote_tp_size}). Use matching TP sizes on prefill and decode." - ) - - # Per-region block_len validation enforcing the P/D invariant. - # REPLICATE regions (MLA, or a whole-model MLA / replicated-KV transfer) - # only allow the number of blocks to differ; SPLIT regions scale with - # the per-rank KV head ratio rather than the raw tp_ratio, because GQA - # replication caps per-rank heads at 1 when tp > total_kv_heads - # (issue #45330). Mamba uses the ssm_sizes counterpart, so skip here. - local_regions = self._transfer_layer_region_indices or range( - len(self.block_len_per_layer) - ) - if self._has_mamba and self.use_mla: - # Hybrid MLA+SSM (e.g. KimiLinear's KDA+MLA): regions are - # kernel-granularity views of the mamba-unified page. The MLA - # per-token page is TP-independent, so the block lengths must - # match up to the kernel block size ratio even under - # heterogeneous TP (remote kernel blocks may be smaller). - # SSM geometry is validated via ssm_sizes/conv offsets instead. - assert self.block_len_per_layer == [ - block_len * block_size_ratio for block_len in nixl_agent_meta.block_lens - ], ( - "Hybrid MLA kernel-granularity block lengths must match " - f"between P and D (block_size_ratio={block_size_ratio}): " - f"local={self.block_len_per_layer}, " - f"remote={nixl_agent_meta.block_lens}." - ) - elif not self._has_mamba: - assert len(local_regions) == len(nixl_agent_meta.block_lens), ( - "Number of KV layers must match between prefill and decode" - ) - model_replicated = self.use_mla or self.transfer_topo.is_kv_replicated( - remote_engine_id - ) - total_kv_heads = self.transfer_topo.total_num_kv_heads - local_heads = self.transfer_topo.local_physical_heads - remote_heads = max(1, total_kv_heads // remote_tp_size) - for i, region_idx in enumerate(local_regions): - local_len = self.block_len_per_layer[region_idx] - replicated = model_replicated or self._is_region_replicated(region_idx) - remote_len = nixl_agent_meta.block_lens[i] - if replicated: - assert local_len // block_size_ratio == remote_len, ( - "KV cache sizes must match between P and D when " - f"replicated (region {i}: local={local_len}, " - f"remote={remote_len}, bsr={block_size_ratio})." - ) - elif tp_ratio > 0: - assert ( - remote_len - == (local_len * remote_heads // local_heads) // block_size_ratio - ), ( - f"SPLIT region {i}: remote P KV block_len {remote_len} " - f"must equal local {local_len} * remote_heads " - f"{remote_heads} // local_heads {local_heads} " - f"// block_size_ratio {block_size_ratio}." - ) - else: - assert block_size_ratio == 1, ( - "Different local/remote block sizes are not supported " - "when P TP > D TP." - ) - assert remote_len == local_len * remote_heads // local_heads, ( - f"SPLIT region {i}: remote P KV block_len {remote_len} " - f"must equal local {local_len} * remote_heads " - f"{remote_heads} // local_heads {local_heads}." - ) - - # TP workers that handhshake with same remote have same #blocks. - assert self.dst_num_blocks[remote_engine_id] == nixl_agent_meta.num_blocks - assert len(nixl_agent_meta.kv_caches_base_addr) == len(local_regions) - num_remote_regions = len(nixl_agent_meta.kv_caches_base_addr) - if nixl_agent_meta.region_num_blocks is not None: - assert len(nixl_agent_meta.region_num_blocks) == num_remote_regions - assert len(nixl_agent_meta.block_strides) == num_remote_regions - assert all( - stride >= block_len - for stride, block_len in zip( - nixl_agent_meta.block_strides, - nixl_agent_meta.block_lens, - strict=True, - ) - ) - if nixl_agent_meta.region_group_ids is not None: - assert len(nixl_agent_meta.region_group_ids) == num_remote_regions - if nixl_agent_meta.region_names is not None: - expected_names = ( - list(self._transfer_layer_names) - if self._transfer_layer_names - else self.region_names - ) - assert nixl_agent_meta.region_names == expected_names, ( - "NIXL transfer regions must name the same model caches in the " - "same order" - ) - if nixl_agent_meta.region_mem_types is not None: - assert len(nixl_agent_meta.region_mem_types) == num_remote_regions - - def sync_recved_kv_to_device(self, req_id: str, meta: ReqMeta): - """Copy recved kv from host buffer to device.""" - assert self.use_host_buffer - assert self.copy_blocks is not None - - local_block_ids = meta.local_physical_block_ids - # Every KV cache group allocates from the same BlockPool, so block ids - # are unique across groups and the per-group copies can be issued as one. - block_ids = [block_id for group in local_block_ids for block_id in group] - # The h2d block copy below is intentionally synchronous. - with gpu_sync_allowed(): - self.copy_blocks( - self.host_xfer_buffers, - self.device_kv_caches, - block_ids, - block_ids, - "h2d", - ) - if logger.isEnabledFor(logging.DEBUG): - logger.debug( - "synced recved kv of request[%s] to device kv buffer," - "local_block_ids: %s. ", - req_id, - ",".join(map(str, local_block_ids)), - ) - - def save_kv_to_host(self, metadata: NixlConnectorMetadata): - """Copy kv from device to host buffer.""" - assert self.use_host_buffer - assert self.copy_blocks is not None - - # The d2h block copies below are intentionally synchronous. - with gpu_sync_allowed(): - for req_id, meta in metadata.reqs_to_save.items(): - meta.local_physical_block_ids = self._logical_to_kernel_block_ids( - meta.local_block_ids, self._physical_blocks_per_logical_kv_block - ) - if logger.isEnabledFor(logging.DEBUG): - logger.debug( - "save_load_kv for request[%s] to host xfer buffer." - "local_block_ids: %s. ", - req_id, - ",".join(map(str, meta.local_physical_block_ids)), - ) - # blocking - block_ids = [ - block_id - for group in meta.local_physical_block_ids - for block_id in group - ] - self.copy_blocks( - self.device_kv_caches, - self.host_xfer_buffers, - block_ids, - block_ids, - "d2h", - ) - - def post_process_device_kv_on_receive( - self, - block_size_ratio: int, - block_ids_list: list[tuple[int, list[int], int]], - convert: bool = True, - ): - """Post process device kv cache after receiving from remote. - - 3 types of conversion supported (``convert``): - * kv_cache_postprocess_layout => convert from HND to NHD - * kv_cache_postprocess_blksize => convert from small block size - to large block size - * kv_cache_postprocess_blksize_and_layout => convert from small - block size to large block size and convert from HND to NHD - - The transfer only covers ``covered_sub_blocks`` remote-sized - sub-blocks of each request's local attention blocks; the rest was - clipped, either by remote-block pairing (block-size ratio) or by the - hetero-ppl front trim in ``_apply_prefix_caching``. Those blocks were - excluded from the scheduler's alloc-time KV zeroing (which would race - the RDMA write), so everything past the covered range is zeroed here. - Stale bytes would otherwise surface as garbage or NaNs once decode - grows into the untransferred tail. - """ - if len(self.device_kv_caches) == 0: - return - assert block_size_ratio >= 1, "Only nP < nD supported currently." - assert self.transfer_topo is not None - if not convert: - logger.debug( - "Post-processing device kv cache on receive by zeroing " - "untransferred blocks." - ) - elif self.enable_permute_local_kv and block_size_ratio > 1: - logger.debug( - "Post-processing device kv cache on receive by converting " - "block_size with %sx bigger and permuting layout from HND" - " to NHD.", - block_size_ratio, - ) - elif self.enable_permute_local_kv: - logger.debug( - "Post-processing device kv cache on receive by permuting layout" - "from HND to NHD." - ) - else: - logger.debug( - "Post-processing device kv cache on receive by converting " - "block_size with %sx bigger.", - block_size_ratio, - ) - - for group, block_ids, covered_sub_blocks in block_ids_list: - attn_caches = [ - self.device_kv_caches[name] - for name in self.kv_cache_config.transfer_groups[group].layer_names - ] - # Blocks the transfer didn't write: the token tail of the last - # partially covered block, then everything beyond it. - covered_blocks, sub_blocks_in_last = divmod( - covered_sub_blocks, block_size_ratio - ) - first_stale = covered_blocks + (1 if sub_blocks_in_last else 0) - has_stale = first_stale < len(block_ids) - indices = None - if convert or has_stale: - indices = async_tensor_h2d(block_ids, attn_caches[0].device, torch.long) - - if convert: - for cache in attn_caches: - if self.enable_permute_local_kv: - kv_postprocess_blksize_and_layout_on_receive( - cache, indices, block_size_ratio - ) - elif KVCacheLayout[self.kv_cache_layout].is_block_contiguous: - # Token-major blocks get the sub-blocks in token order. - kv_postprocess_blksize_on_receive( - cache, indices, block_size_ratio - ) - - if sub_blocks_in_last: - last_block_id = block_ids[covered_blocks] - for cache in attn_caches: - # Attention caches are [B, H, N, C]: tokens on dim 2. - sub_block_tokens = cache.shape[2] // block_size_ratio - zero_from = sub_blocks_in_last * sub_block_tokens - cache[last_block_id, :, zero_from:].zero_() - if has_stale: - assert indices is not None - stale_ids = indices[first_stale:] - for cache in attn_caches: - cache.index_fill_(0, stale_ids, 0) - - def post_process_device_kv_on_receive_heterogeneous_attn( - self, block_ids: list[int] - ): - """Post process device kv cache after receiving from remote - for heterogeneous attention. - """ - assert self.enable_heterogeneous_attn_post_process - - indices = torch.tensor(block_ids, device=self.device_type, dtype=torch.long) - - for _, cache_or_caches in self.device_kv_caches.items(): - current_platform.pack_kv_cache( - kv_cache=cache_or_caches, - indices=indices, - ) - - def _leaves_dram_xfer_to_rank0(self) -> bool: - """Whether this rank leaves the DRAM regions' NIXL work to TP rank 0. - - A HiSparse host pool shared by the local TP ranks (one mmap per DP - rank) is registered with NIXL and filled by TP rank 0 alone: every - rank would otherwise register the same pages on every NIC and READ the - same host KV into them. Private pools are unaffected. - """ - if not ( - self._mixed_mem_types and self.kv_cache_config.hisparse_shared_host_pool - ): - return False - skip = self.tp_rank != 0 - logger.info( - "HiSparse host pool: NIXL DRAM registration and reads on TP rank 0 " - "only (tp_rank=%d, skip=%s)", - self.tp_rank, - skip, - ) - return skip - - def _zero_region_blocks(self, block_ids: BlockIds) -> None: - """Clear clipped physical pages in their owning region only.""" - if not any(block_ids): - return - bases = self.kv_caches_base_addr[self.engine_id][self.tp_rank] - for region, blocks in enumerate(block_ids): - if not blocks: - continue - if self._skip_dram_xfer and self.region_mem_types[region] == "DRAM": - # The shared pool is rank 0's to write. - continue - cache = self.device_kv_caches[self.region_names[region]] - storage = cache.untyped_storage() - pages = torch.empty(0, dtype=torch.uint8, device=cache.device).set_( - storage, - bases[region] - storage.data_ptr(), - (self.region_num_blocks[region], self.block_len_per_layer[region]), - (self.block_stride_per_layer[region], 1), - ) - for block in blocks: - pages[block].zero_() - - def get_transfer_results(self) -> KVConnectorTransferResults: - """Get transfers that completed on this specific worker. - - The scheduler process (via the MultiprocExecutor) will use this output - to track which workers are done. - """ - assert self.transfer_topo is not None - done_sending = self._get_new_notifs() - done_recving, newly_failed = self._pop_done_transfers(self._recving_transfers) - if newly_failed: - self._recv_failures.update(newly_failed) - - done_sending.update(self._replicated_pcp_done_sending) - self._replicated_pcp_done_sending.clear() - - # Process receive failures reported by background threads. Each - # producer already recorded a specific metric (handshake or - # notification), so only the failure bookkeeping runs here. - while not self._failed_recv_reqs.empty(): - try: - req_id = self._failed_recv_reqs.get_nowait() - except queue.Empty: - break - self._handle_failed_transfer( - req_id, None, self._recv_failures, record_failed_transfer=False - ) - - failed_recv_reqs: set[ReqId] = set() - if self._recv_failures: - # Keep failure bookkeeping independent of healthy request count. - for req_id in tuple(self._recv_failures): - if req_id not in self._recving_metadata: - self._recv_failures.remove(req_id) - elif req_id not in self._recving_transfers: - self._recv_failures.remove(req_id) - failed_recv_reqs.add(req_id) - done_recving.update(failed_recv_reqs) - - if len(done_sending) > 0 or len(done_recving) > 0: - logger.debug( - "Rank %s, get_finished: %s requests done sending " - "and %s requests done recving (%s failed)", - self.tp_rank, - len(done_sending), - len(done_recving), - len(failed_recv_reqs), - ) - - block_ids_for_blocksize_post_process = defaultdict(list) - block_ids_for_heterogeneous_attn_post_process = list[list[int]]() - direct_device_recving = set[str]() - for req_id in done_recving: - # clean up metadata for completed requests - meta = self._recving_metadata.pop(req_id, None) - assert meta is not None, f"{req_id} not found in recving_metadata list" - - # Skip KV sync and post-processing for failed requests - if req_id in failed_recv_reqs: - self._pending_recv_notifs.pop(req_id, None) - # TODO (NickLucche) handle failed transfer for HMA. - if not self._is_hma_required: - self._invalid_block_ids.put(set(meta.local_block_ids[0])) - logger.warning( - "Skipping KV post-processing for failed request %s", - req_id, - ) - continue - - self._send_pending_recv_notifs(req_id) - assert meta.remote is not None - if self.use_host_buffer: - self.sync_recved_kv_to_device(req_id, meta) - - direct_device_recving.add(req_id) - - if meta.region_blocks_to_zero is not None: - # P/D group positions differ. Use the actual region read plan, - # including any local allocation padding the read did not cover. - self._zero_region_blocks(meta.region_blocks_to_zero) - continue - - # Post processing for heteroblocksize/layout, and for blocks the - # transfer clipped. The latter happens either at remote-block - # granularity (block_size_ratio > 1) or at kernel-block - # granularity, when equal kernel pages meet differing logical - # block sizes and _apply_prefix_caching front-trims to the - # minimum count (hybrid heterogeneous TP). - remote_info = self.transfer_topo.get_engine_info(meta.remote.engine_id) - block_size_ratio = self.transfer_topo.block_size_ratio( - remote_info.remote_block_size - ) - hetero_ppl = ( - remote_info.remote_physical_blocks_per_logical - != self._physical_blocks_per_logical_kv_block - ) - if block_size_ratio > 1 or self.enable_permute_local_kv or hetero_ppl: - for g, local_group in enumerate(meta.local_physical_block_ids): - if not local_group or _is_ssm_spec(self._group_spec_types[g]): - continue - # Number of remote-sized sub-blocks the transfer covered; - # everything past this was clipped and must be zeroed. - covered_sub_blocks = min( - len(local_group) * block_size_ratio, - len(meta.remote.block_ids[g]), - ) - block_ids_for_blocksize_post_process[block_size_ratio].append( - (g, local_group, covered_sub_blocks) - ) - # post processing for heterogeneous attention - if self.enable_heterogeneous_attn_post_process: - block_ids_for_heterogeneous_attn_post_process.append( - meta.local_physical_block_ids[0] - ) - for ( - block_size_ratio, - block_ids_list, - ) in block_ids_for_blocksize_post_process.items(): - # MLA never needs the block-size/layout conversion, but its - # clipped blocks still need zeroing. - convert = not self.use_mla and ( - block_size_ratio > 1 or self.enable_permute_local_kv - ) - self.post_process_device_kv_on_receive( - block_size_ratio, block_ids_list, convert - ) - - for block_ids in block_ids_for_heterogeneous_attn_post_process: - self.post_process_device_kv_on_receive_heterogeneous_attn(block_ids) - - self._sync_device_after_direct_recv(direct_device_recving) - - # Handle timeout to avoid stranding blocks on remote. - self._reap_expired_send_leases(done_sending) - - self._cleanup_remote_engines() - return KVConnectorTransferResults( - finished_sending=done_sending, - finished_recving=done_recving, - failed_recving=failed_recv_reqs, - ) - - def get_finished(self) -> tuple[set[str], set[str]]: - """Compatibility wrapper for the legacy completion API.""" - results = self.get_transfer_results() - return results.finished_sending, results.finished_recving - - def _sync_device_after_direct_recv(self, done_recving: set[str]) -> None: - """Make direct NIXL writes visible before model execution.""" - requires_sync = current_platform.is_rocm() and self._has_mamba - if self.use_host_buffer or not done_recving or not requires_sync: - return - - torch.accelerator.synchronize() - - def _get_new_notifs(self) -> set[str]: - """Get req_ids which got a remote xfer notification. - - Subclasses must implement this to handle mode-specific notifications. - """ - raise NotImplementedError - - def _reap_expired_send_leases(self, done_sending: set[str]) -> None: - """Reclaim expired send-side KV leases into ``done_sending``. - - ``_reqs_to_send`` is not ordered by expiry: heartbeats update the - deadline in place, and mixed TTLs share the map, so a live head - entry can sit in front of already-expired ones. Scan every entry - rather than stopping at the first still-live request. - """ - if not self._reqs_to_send: - return - now = time.perf_counter() - expired = [ - req_id for req_id, expires in self._reqs_to_send.items() if now >= expires - ] - for req_id in expired: - count = self.consumer_notification_counts_by_req.pop(req_id, 0) - self.expected_consumer_notifications_by_req.pop(req_id, None) - self.xfer_stats.record_kv_expired_req() - logger.warning( - "Releasing expired KV blocks for request %s which were " - "retrieved by %d remote worker(s) before lease expired.", - req_id, - count, - ) - self._reqs_to_process.remove(req_id) - del self._reqs_to_send[req_id] - done_sending.add(req_id) - - def _handle_heartbeat(self, payload: str) -> None: - """Extend leases for requests referenced in a heartbeat. - - Args: - payload: comma-separated P-side request IDs, e.g. - "req_abc,req_def". - - """ - new_expiry = time.perf_counter() + self._lease_extension - for req_id in payload.split(","): - if req_id in self._reqs_to_send: - old = self._reqs_to_send[req_id] - self._reqs_to_send[req_id] = max(old, new_expiry) - logger.debug( - "Heartbeat extended lease for request %s " - "by %ds (old_expiry=%.1f, new_expiry=%.1f)", - req_id, - self._lease_extension, - old, - new_expiry, - ) - - def _pop_done_transfers( - self, transfers: dict[str, list[int]] - ) -> tuple[set[str], set[str]]: - """Poll transfers, retaining handles until they can be released. - - Args: - transfers: Outstanding handles by request, updated in place. - - Returns: - Requests with no outstanding handles and requests with observed - failures. Failed requests may still have outstanding handles. - - """ - done_req_ids: set[str] = set() - failed_req_ids: set[str] = set() - for req_id, handles in list(transfers.items()): - in_progress = [] - for handle in handles: - try: - xfer_state = self.nixl_wrapper.check_xfer_state(handle) - if xfer_state == "DONE": - res = self.nixl_wrapper.get_xfer_telemetry(handle) - self.xfer_stats.record_transfer(res) - self.nixl_wrapper.release_xfer_handle(handle) - elif xfer_state == "PROC": - in_progress.append(handle) - else: - self._log_failure( - failure_type="transfer_failed", - req_id=req_id, - xfer_state=xfer_state, - ) - if not self._handle_failed_transfer( - req_id, handle, failed_req_ids - ): - in_progress.append(handle) - except Exception as e: - self._log_failure( - failure_type="transfer_exception", - req_id=req_id, - error=e, - ) - if not self._handle_failed_transfer(req_id, handle, failed_req_ids): - in_progress.append(handle) - - if not in_progress: - done_req_ids.add(req_id) - del transfers[req_id] - else: - transfers[req_id] = in_progress - return done_req_ids, failed_req_ids - - def _try_release_xfer_handle(self, req_id: str, handle: int) -> bool: - """Release a handle, returning False if the caller must retain it.""" - try: - self.nixl_wrapper.release_xfer_handle(handle) - except Exception as e: - # A status error does not guarantee that the backend stopped DMA. - self._log_failure( - failure_type="transfer_release_failed", - msg="Retaining handle and blocks until release succeeds", - req_id=req_id, - error=e, - ) - return False - return True - - def _handle_failed_transfer( - self, - req_id: str, - handle: int | None, - failed_req_ids: set[str] | None = None, - record_failed_transfer: bool = True, - ) -> bool: - """Record a failure and release its handle, returning False to retain it. - - Args: - req_id: The request ID. - handle: The transfer handle. - failed_req_ids: Requests observed as failed; each stays listed - until it has no outstanding handles. - record_failed_transfer: Whether to count the failure toward the - transport-failure metric. Callers that already recorded a more - specific metric (eg KV expiry, reported separately from - transport failures) pass False. - - """ - if record_failed_transfer: - self.xfer_stats.record_failed_transfer() - if ( - self._TRANSFER_MODE == "pull" - and (meta := self._recving_metadata.get(req_id)) is not None - and meta.remote is not None - ): - self._failed_remote_engines.add(meta.remote.engine_id) - if failed_req_ids is not None: - failed_req_ids.add(req_id) - return handle is None or self._try_release_xfer_handle(req_id, handle) - - def _send_pending_recv_notifs(self, req_id: str) -> None: - """Send notifications deferred by split DRAM/VRAM reads.""" - for agent_name, notif_id in self._pending_recv_notifs.pop(req_id, []): - try: - self.nixl_wrapper.send_notif(agent_name, notif_msg=notif_id) - except Exception as e: - self._log_failure( - failure_type="notification_failed", - msg="P worker blocks will be freed after timeout. " - "This may indicate network issues.", - req_id=req_id, - error=e, - remote_agent_name=agent_name, - ) - self.xfer_stats.record_failed_notification() - - def _send_heartbeats(self, metadata: NixlConnectorMetadata) -> None: - """Send heartbeats to remote engines, extending the lease on KV blocks.""" - for engine_id, hb_info in metadata.heartbeat_by_engine.items(): - # Proactive handshake (this request may still be in waiting queue) so - # the **next** heartbeat for this remote can go through. - if ( - self._ensure_handshake( - engine_id, - hb_info.host, - hb_info.port, - hb_info.tp_size, - hb_info.dcp_size, - hb_info.pp_size, - self._hb_handshake_notif_only and hb_info.pp_size > 1, - ) - is not None - ): - continue # handshake is still pending - - # Refresh the TTL: a heartbeat means the router still paired this P-D, - # so requests are still waiting on this engine - self._engine_last_active[engine_id] = time.perf_counter() - # Build the heartbeat message: "HB:req1,req2,..." - hb_msg = ("HB:" + ",".join(hb_info.req_ids)).encode() - for agent_name in self._remote_agents[engine_id].values(): - try: - self.nixl_wrapper.send_notif(agent_name, notif_msg=hb_msg) - except Exception: - logger.debug( - "Failed to send heartbeat to engine %s", - engine_id, - exc_info=True, - ) - - def get_mapped_blocks( - self, block_ids: np.ndarray, block_size_ratio: int - ) -> np.ndarray: - """Calculates the new set of block IDs by mapping every element - in the (potentially sparse) input array. - Example: block_ids=[0, 2], block_size_ratio=2 - get_mapped_blocks 0 1 [2 3] 4 5 - # remote is |h0-b0|h1-b0||h0-b1|h1-b1||h0-b1|h1-b1|| - # local is |h0-b0......||h1-b0......||h2-b0........ - local_block_ids 0 [1] 2 - """ - if block_ids.size == 0: - return np.array([], dtype=np.int64) - - start_ids = block_ids * block_size_ratio - offsets = np.arange(block_size_ratio) - mapped_2d = start_ids[:, None] + offsets[None, :] - - return mapped_2d.flatten().astype(np.int64) - - def _map_block_ids_for_block_size_ratio( - self, - local_block_ids: BlockIds, - remote_block_ids: BlockIds, - block_size_ratio: int, - ) -> tuple[BlockIds, BlockIds]: - """Map attention-group block ids to remote-block granularity. - - Each local attention block is split into ``block_size_ratio`` - sub-blocks paired 1:1 with remote blocks. Sub-blocks beyond the - remote list — the untransferred tail of the last local block — are - clipped here and zeroed in the receive post-process. Mamba state - blocks are indivisible and transfer 1:1, unexpanded. - - ex: remote (prefill) block ids with block_size 4: - [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] - Local (decode) block ids with block_size 16: [1, 2, 3] expand to - [4, 5, ..., 15], then clip to the first 10 to pair 1:1 with remote. - """ - mapped_local: list[list[int]] = [] - mapped_remote: list[list[int]] = [] - for i, remote_group in enumerate(remote_block_ids): - local_group = local_block_ids[i] if local_block_ids else [] - if _is_ssm_spec(self._group_spec_types[i]): - mapped_local.append(list(local_group)) - mapped_remote.append(list(remote_group)) - continue - mapped = self.get_mapped_blocks( - np.asarray(local_group), block_size_ratio - ).tolist() - if len(mapped) > len(remote_group): - mapped = mapped[: len(remote_group)] - mapped_local.append(mapped) - mapped_remote.append(list(remote_group)) - if not any(mapped_local): - # Full prefix cache hit is indicated with an empty list. - return [], mapped_remote - return mapped_local, mapped_remote - - def _logical_to_kernel_block_ids(self, block_ids: BlockIds, ratio: int) -> BlockIds: - """Convert block ids to kernel physical block ids. - This is required when the logical block size (the one set by the user) - does not match the one required by the attn backend. - `ratio` is the number of physical blocks per logical block. - We always receive logical blocks from the engine, so we expand them here eg: - logical block ids: [(SW-clipped) [1], (FA) [2, 3]], ratio=2 - physical block ids: [(SW-clipped) [2, 3], (FA) [4, 5, 6, 7]] - """ - if ratio == 1: - # Noop when physical and logical block sizes are the same - return block_ids - block_arange = np.arange(0, ratio).reshape(1, -1) - # Mamba blocks have no logical<>physical discrepancy (block-size=1) - physical_block_ids = [] - for i, group in enumerate(block_ids): - spec = self.kv_cache_config.transfer_groups[i].kv_cache_spec - if _is_ssm_spec(get_representative_spec_type(spec)): - physical_block_ids.append(group) - else: - physical_block_ids.append( - (np.array(group)[:, None] * ratio + block_arange) - .reshape(-1) - .tolist() - ) - return physical_block_ids - - def _apply_dcp_prefix_caching( - self, - local_ids: list[int], - remote_ids: list[int], - remote_rank: int, - local_dcp_size: int, - local_dcp_rank: int, - remote_dcp_size: int, - local_num_computed_blocks: int, - num_remote_blocks: int | None = None, - ) -> tuple[list[int], list[int]]: - """Match local and remote block IDs by global DCP position. - - ``local_ids`` excludes blocks already satisfied by the local prefix - cache, while ``remote_ids`` contains the full transferable remote list. - ``num_remote_blocks``, when provided, is the global block count before - DCP partitioning and excludes allocation padding. - - Example: - local DCP size 2, rank 0 owns *global positions*=[0, 2, 4, 6] - cached_blocks=[0,2,4] - to transfer: [6] (the rest is cached) - - remote DCP size 4 - remote rank 0 owns [0, 4] -> no more match, skip the read - remote rank 2 owns [2, 6] -> match position 6 and read - - An empty result (rank 0 above) takes the notification-only path, allowing the - remote to release its blocks without performing a transfer. - - """ - local_size, remote_size = local_dcp_size, remote_dcp_size - if num_remote_blocks is not None: - num_local = cdiv(num_remote_blocks - local_dcp_rank, local_size) - local_ids = local_ids[: max(0, num_local - local_num_computed_blocks)] - - if local_size == remote_size: - local_slice = local_ids - remote_slice = remote_ids[local_num_computed_blocks:] - elif local_size < remote_size: - k = remote_size // local_size - p = (remote_rank - local_dcp_rank) // local_size - start_local = (p - local_num_computed_blocks) % k - start_remote = (local_num_computed_blocks + start_local - p) // k - local_slice = local_ids[start_local::k] - remote_slice = remote_ids[start_remote:] - else: - k = local_size // remote_size - remote_dcp_rank = remote_rank % remote_size - c = (local_dcp_rank - remote_dcp_rank) // remote_size - start_remote = c + local_num_computed_blocks * k - local_slice = local_ids - remote_slice = remote_ids[start_remote::k] - - matched_blocks = min(len(local_slice), len(remote_slice)) - if num_remote_blocks is not None and matched_blocks < len(local_slice): - raise ValueError("Remote KV pages do not cover the requested range") - return local_slice[:matched_blocks], remote_slice[:matched_blocks] - - @staticmethod - def _block_ids_by_region( - block_ids: BlockIds, region_group_ids: list[int] - ) -> BlockIds: - """Expand group block IDs into the corresponding region order.""" - shared_block_ids = list(itertools.chain.from_iterable(block_ids)) - block_ids_by_region = [] - for group_id in region_group_ids: - if group_id == _SHARED_REGION_GROUP_ID: - block_ids_by_region.append(shared_block_ids.copy()) - continue - block_ids_by_region.append(list(block_ids[group_id])) - return block_ids_by_region - - def _apply_prefix_caching_by_region( - self, - decode_block_ids: BlockIds, - prefill_block_ids: BlockIds, - *, - num_computed_blocks: list[int] | None = None, - num_remote_blocks: int | None = None, - remote_rank: int = 0, - remote_dcp_size: int = 1, - ) -> tuple[BlockIds, BlockIds]: - """Pair an uncached decode suffix with the same prefill regions.""" - assert len(decode_block_ids) == len(prefill_block_ids) - if not any(decode_block_ids): - empty_regions: list[list[int]] = [[] for _ in decode_block_ids] - return empty_regions, empty_regions.copy() - - if num_computed_blocks is not None: - assert num_remote_blocks is not None - matched_decode, matched_prefill = [], [] - for decode_region, prefill_region, start in zip( - decode_block_ids, prefill_block_ids, num_computed_blocks, strict=True - ): - local, remote = self._apply_dcp_prefix_caching( - decode_region, - prefill_region, - remote_rank, - self.dcp_size, - self.dcp_rank, - remote_dcp_size, - start, - num_remote_blocks=num_remote_blocks, - ) - matched_decode.append(local) - matched_prefill.append(remote) - return matched_decode, matched_prefill - - trimmed_prefill: list[list[int]] = [] - for decode_region, prefill_region in zip( - decode_block_ids, prefill_block_ids, strict=True - ): - if len(decode_region) > len(prefill_region): - raise ValueError( - "Decode allocated more KV pages than the prefill worker supplied" - ) - trimmed_prefill.append( - list(prefill_region[-len(decode_region) :]) if decode_region else [] - ) - return decode_block_ids, trimmed_prefill - - def _apply_prefix_caching( - self, - decode_block_ids: BlockIds, - prefill_block_ids: BlockIds, - decode_physical_per_logical: int, - prefill_physical_per_logical: int, - ) -> tuple[BlockIds, BlockIds]: - """Trim block ID lists so only the uncomputed suffix is transferred. - - Inputs are *kernel* (physical) block IDs, already expanded from logical IDs - with each side's physical-per-logical ratio. Both pull and push call this after - that expansion so the trim happens at kernel granularity. - - The prefix-cache hit is always on the decode (D) side, so ``decode`` - holds only its uncomputed blocks while prefill (P) holds the full - sequence. This is mode-independent: pull passes its own (D) blocks as - ``decode``, push passes the remote D registration as ``decode``. - - For non-Mamba models: end-trim ``prefill`` to match ``decode`` count, so - already-cached prefix blocks are skipped in the transfer. - - For Mamba hybrid: SSM groups pair state slots by position and FA - groups end-trim to the uncomputed suffix when physical-per-logical - matches. - """ - # Partial prefix cache hit: just transfer uncomputed blocks. - # Skip mamba groups — their blocks represent full state (conv+ssm), - # not per-token data, so trimming would corrupt the transfer. - prefill_block_ids = list(prefill_block_ids) - if not self._has_mamba: - for i, prefill_group in enumerate(prefill_block_ids): - num_decode_blocks = len(decode_block_ids[i]) - assert num_decode_blocks <= len(prefill_group) - if num_decode_blocks < len(prefill_group): - prefill_block_ids[i] = prefill_group[-num_decode_blocks:] - else: - # (NOTE: ZhanqiuHu) HeteroTP can cause different kernel block counts - # due to logical block rounding. - # Example: 640 prompt tokens, kernel_block_size=64 - # prefill physical_per_logical=10, decode physical_per_logical=6 - # prefill logical ids from kv_transfer_params = [0] - # decode logical ids allocated = [0, 1] - # prefill kernel blocks: [0..9] (1*10=10) - # decode kernel blocks: [0..11] (2*6=12) - # actual data blocks = ceil(640/64) = 10, trim both to 10 - # Vice versa (prefill physical_per_logical=6, decode=10): - # prefill logical ids = [0, 1], decode logical ids = [0] - # prefill kernel blocks: [0..11] (2*6=12) - # decode kernel blocks: [0..9] (1*10=10) - # actual data blocks = ceil(640/64) = 10, trim both to 10 - decode_block_ids = list(decode_block_ids) - for i, prefill_group in enumerate(prefill_block_ids): - num_decode_blocks = len(decode_block_ids[i]) - num_prefill_blocks = len(prefill_group) - if _is_ssm_spec(self._group_spec_types[i]): - if num_decode_blocks == num_prefill_blocks: - continue - # Only state-bearing slots reach here, normally just one - # (see get_exchange_clipped_blocks). A longer prefill list - # carries earlier positions the decode side already has - # (prefix hit) -> take its tail; a longer decode list holds - # the position D recomputes itself, which gets no prefill - # state. - assert num_decode_blocks - num_prefill_blocks <= 1, ( - f"Group {i}: unpairable SSM state slots, " - f"decode={num_decode_blocks} prefill={num_prefill_blocks}" - ) - num_blocks = min(num_decode_blocks, num_prefill_blocks) - if num_decode_blocks < num_prefill_blocks: - prefill_block_ids[i] = prefill_group[-num_blocks:] - else: - decode_block_ids[i] = decode_block_ids[i][:num_blocks] - elif ( - decode_physical_per_logical == prefill_physical_per_logical - and num_decode_blocks < num_prefill_blocks - ): - # Partial prefix cache hit for FA group. - prefill_block_ids[i] = prefill_group[-num_decode_blocks:] - else: - # TODO Handle prefix caching with different block_sizes - # Allocation rounding legitimately leaves up to - # ppl - 1 trailing dead kernel blocks per side (plus one - # extra decode block for the recomputed final token), so - # the counts may differ by up to the sum of the two - # ratios; anything larger indicates mismatched lists. - max_padding = ( - decode_physical_per_logical + prefill_physical_per_logical - ) - assert abs(num_decode_blocks - num_prefill_blocks) <= max_padding, ( - f"Group {i}: |{num_decode_blocks} - " - f"{num_prefill_blocks}| > {max_padding}" - ) - num_blocks = min(num_decode_blocks, num_prefill_blocks) - decode_block_ids[i] = decode_block_ids[i][:num_blocks] - prefill_block_ids[i] = prefill_group[:num_blocks] - return decode_block_ids, prefill_block_ids - - def get_kv_connector_stats(self) -> KVConnectorStats | None: - """Get the KV transfer stats for the connector.""" - # Clear stats for next iteration - if not self.xfer_stats.is_empty(): - return self.xfer_stats.clone_and_reset() - return None - - def get_block_ids_with_load_errors(self) -> set[int]: - """Return and clear the set of block IDs that failed to load. - - This is called by the scheduler to identify blocks that need - to be retried after a NIXL transfer failure. - """ - # Drain the queue (thread-safe, no lock needed). - result: set[int] = set() - while not self._invalid_block_ids.empty(): - try: - result.update(self._invalid_block_ids.get_nowait()) - except queue.Empty: - break - return result - - def _evict_stale_engines(self) -> None: - """Scan for and evict remote engines that have exceeded their TTL. - - Called from the main thread in when a new remote engine appears. - We can only go OOM as we discover and register a new remote, therefore we make - sure we clean up stale engine data structures before then. This invariant - prevents us from using background threads, though memory usage is not guaranteed - to be "optimal" until a new handshake is performed. - - Engines with active transfers, heartbeats or pending handshakes cannot - be stale: - - Reads stamp _engine_last_active when they are issued, and engines a - transfer is still reading from are held back explicitly, since the - stamp is not refreshed while the read runs. - - Heartbeats stamp it too, so an engine stays registered while this - instance holds requests waiting on it. - - Pending handshakes don't have an _engine_last_active entry yet - """ - # NOTE (NickLucche): This does NOT currently prevent OOMing if a huge number - # of remote engines is registered all at once (adding a background cleanup - # thread wouldnt help either). - # If that scenario is plausible, we can follow up with an LRU eviction policy. - if self._engine_ttl <= 0: - return - - now = time.perf_counter() - busy = self._engines_with_inflight_transfers() - for eid, last_active in list(self._engine_last_active.items()): - if now - last_active > self._engine_ttl and eid not in busy: - self._cleanup_remote_engine(eid) - - def _engines_with_inflight_transfers(self) -> set[EngineId]: - """Remote engines a transfer is still reading from. - - The timestamp is stamped when a read is issued and not refreshed while - it runs, so a transfer that outlives the TTL leaves its engine looking - idle. A peer that has lost its NIC holds one indefinitely. - """ - return { - meta.remote.engine_id - for req_id in self._recving_transfers - if (meta := self._recving_metadata.get(req_id)) is not None - and meta.remote is not None - } - - def _track_remote_engine_replacement( - self, engine_id: EngineId, host: str, port: int - ) -> None: - """Record the engine currently serving a pull peer address.""" - # TODO: Also handle push mode, which handshakes in both directions. - if self._TRANSFER_MODE != "pull": - return - previous = self._engine_by_address.get((host, port)) - self._engine_by_address[(host, port)] = engine_id - if previous is None or previous == engine_id: - return - logger.info( - "Remote engine at %s:%d changed from %s to %s.", - host, - port, - previous, - engine_id, - ) - - def _cleanup_remote_engines(self) -> None: - """Release replaced or locally invalid pull peers after their reads drain.""" - # TODO: Also handle push mode, which handshakes in both directions. - if self._TRANSFER_MODE != "pull": - return - # A background handshake may be loading native metadata with the GIL - # released. Defer both probing and cleanup until it has finished. - with self._handshake_lock: - if self._handshake_futures: - return - replaced = self._remote_agents.keys() - self._engine_by_address.values() - now = time.perf_counter() - for engine_id in self._failed_remote_engines - self._invalid_remote_engines: - agents = self._remote_agents.get(engine_id, {}).values() - try: - # No descriptors: query local existence, not peer health. - if not all(map(self.nixl_wrapper.check_remote_metadata, agents)): - self._mark_remote_engine_invalid(engine_id, now) - except Exception: - logger.warning( - "Could not check local NIXL metadata for engine %s.", - engine_id, - exc_info=True, - ) - self._failed_remote_engines.clear() - # Invalid peers keep rejecting new reads until their backoff has elapsed. - invalid = { - engine_id - for engine_id in self._invalid_remote_engines - if self._invalid_engine_backoff.get(engine_id, (0.0, 0.0))[1] <= now - } - if not replaced and not invalid: - return - # _recving_metadata is only accessed from this (main) thread. - busy = { - meta.remote.engine_id - for meta in self._recving_metadata.values() - if meta.remote is not None - } - # Replaced peers can still serve queued requests. Invalid peers reject - # new reads, but must retain descriptors until outstanding handles end. - invalid -= self._engines_with_inflight_transfers() - for engine_id in (replaced - busy) | invalid: - self._cleanup_remote_engine(engine_id, log_eviction=False) - logger.info( - "Released NIXL state for replaced or locally invalid remote engine %s.", - engine_id, - ) - - def _mark_remote_engine_invalid(self, engine_id: EngineId, now: float) -> None: - """Schedule release of an engine whose local NIXL metadata is gone. - - The first invalidation is released as soon as reads drain. Repeats - before the peer has stayed healthy for the max backoff double the - delay, during which reads fail fast instead of re-handshaking. - """ - # Forget engines that have stayed healthy long enough. - for eid, (_, release_at) in list(self._invalid_engine_backoff.items()): - if now - release_at > _INVALID_PEER_MAX_BACKOFF_S: - del self._invalid_engine_backoff[eid] - prev = self._invalid_engine_backoff.get(engine_id) - backoff = 0.0 - if prev is not None: - backoff = min( - max(2 * prev[0], _INVALID_PEER_MIN_BACKOFF_S), - _INVALID_PEER_MAX_BACKOFF_S, - ) - self._invalid_engine_backoff[engine_id] = (backoff, now + backoff) - self._invalid_remote_engines.add(engine_id) - logger.warning( - "Local NIXL metadata for remote engine %s was invalidated; " - "re-handshaking after reads drain%s.", - engine_id, - f" and a {backoff:.0f}s backoff" if backoff else "", - ) - - def _cleanup_remote_engine( - self, engine_id: EngineId, *, log_eviction: bool = True - ) -> None: - """Remove all state for a single remote engine. - - Releases NIXL resources (dlist handles, remote agents) and clears - all per-engine data structures. Used by TTL eviction, replaced/invalid - peer cleanup and shutdown. - """ - assert engine_id in self._remote_agents - - # Notif-only engines (push-mode D side) have no descriptor state. - for handle in self.dst_xfer_side_handles.pop(engine_id, {}).values(): - self.nixl_wrapper.release_dlist_handle(handle) - # Pop under the handshake lock; NIXL teardown stays outside it. - with self._handshake_lock: - agents = self._remote_agents.pop(engine_id) - for address, eid in list(self._engine_by_address.items()): - if eid == engine_id: - del self._engine_by_address[address] - for agent_name in agents.values(): - self.nixl_wrapper.remove_remote_agent(agent_name) - - self._failed_remote_engines.discard(engine_id) - self._invalid_remote_engines.discard(engine_id) - - self.kv_caches_base_addr.pop(engine_id, None) - self.dst_num_blocks.pop(engine_id, None) - self.dst_region_num_blocks.pop(engine_id, None) - self.dst_region_group_ids.pop(engine_id, None) - self.dst_uses_region_group_mapping.pop(engine_id, None) - self.dst_region_mem_types.pop(engine_id, None) - self.tp_mappings.pop(engine_id, None) - if self.transfer_topo is not None: - self.transfer_topo.unregister_remote_engine(engine_id) - - # Drop the cached clock offset; it is re-measured on the next handshake. - self._engine_clock_offset.pop(engine_id, None) - # A just-completed handshake may not have recorded activity yet, so - # tolerate a missing entry. - last_active = self._engine_last_active.pop(engine_id, None) - if log_eviction and last_active is not None: - logger.info( - "Evicted stale remote engine %s (inactive for %.1fs).", - engine_id, - time.perf_counter() - last_active, - ) - - def __del__(self): - with contextlib.suppress(Exception): - self.shutdown() - - def _finish_shutdown(self) -> None: - self._recving_transfers.clear() - try: - for handle in self.src_xfer_handles_by_block_size.values(): - self.nixl_wrapper.release_dlist_handle(handle) - for handles in self.src_xfer_handles_by_tp_ratio.values(): - for handle in handles: - self.nixl_wrapper.release_dlist_handle(handle) - for dram_handles in self._dram_src_handles_by_tp_ratio.values(): - for dram_handle in dram_handles: - if dram_handle is not None: - self.nixl_wrapper.release_dlist_handle(dram_handle) - for dram_handle in self._dram_src_handles_by_block_size.values(): - if dram_handle is not None: - self.nixl_wrapper.release_dlist_handle(dram_handle) - except Exception: - logger.exception("NIXL dlist-handle release failed at shutdown.") - self.src_xfer_handles_by_block_size.clear() - self.src_xfer_handles_by_tp_ratio.clear() - self._dram_src_handles_by_tp_ratio.clear() - self._dram_src_handles_by_block_size.clear() - try: - for engine_id in list(self._remote_agents): - self._cleanup_remote_engine(engine_id, log_eviction=False) - except Exception: - logger.exception("NIXL remote-engine cleanup failed at shutdown.") - try: - for desc in self._registered_descs: - self.nixl_wrapper.deregister_memory(desc) - finally: - self._registered_descs.clear() - # Drop cache references before their owners release registered - # host memory; handshake futures may outlive model-runner shutdown. - self.device_kv_caches = {} - self.host_xfer_buffers = {} - - def shutdown(self) -> None: - """Shutdown the connector worker.""" - if not hasattr(self, "_handshake_initiation_executor"): - # error happens during init, no need to shutdown - return - self._handshake_initiation_executor.shutdown(wait=False) - for handles in self._recving_transfers.values(): - for handle in handles: - self.nixl_wrapper.release_xfer_handle(handle) - self._recving_transfers.clear() - self._finish_shutdown() diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/connector.py deleted file mode 100644 index 5511edc580..0000000000 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/connector.py +++ /dev/null @@ -1,426 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""NIXL connector facades. - -This module hosts the thin facade classes that vLLM's KV-connector layer -instantiates. Almost all the real work lives in the per-mode scheduler -and worker classes; the connector classes here only forward calls. - -* :class:`NixlBaseConnector` – common logic shared by pull and push. -* :class:`NixlPullConnector` – pull-based (READ) KV transfer. -* :class:`NixlPushConnector` – push-based (WRITE) KV transfer. -* ``NixlConnector`` – backward-compatible alias for :class:`NixlPullConnector`. -""" - -from typing import TYPE_CHECKING, Any - -import torch - -from vllm.config import KVTransferConfig, VllmConfig -from vllm.distributed.kv_transfer.kv_connector.utils import ( - EngineId, -) -from vllm.distributed.kv_transfer.kv_connector.v1.base import ( - CopyBlocksOp, - KVConnectorBase_V1, - KVConnectorHandshakeMetadata, - KVConnectorMetadata, - KVConnectorRole, - KVConnectorTransferResults, - SupportsHMA, -) -from vllm.distributed.kv_transfer.kv_connector.v1.metrics import ( - KVConnectorPromMetrics, - KVConnectorStats, - PromMetric, - PromMetricT, -) -from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import ( - NixlConnectorMetadata, -) -from vllm.distributed.kv_transfer.kv_connector.v1.nixl.pull_scheduler import ( - NixlPullConnectorScheduler, -) -from vllm.distributed.kv_transfer.kv_connector.v1.nixl.pull_worker import ( - NixlPullConnectorWorker, -) -from vllm.distributed.kv_transfer.kv_connector.v1.nixl.push_scheduler import ( - NixlPushConnectorScheduler, -) -from vllm.distributed.kv_transfer.kv_connector.v1.nixl.push_worker import ( - NixlPushConnectorWorker, -) -from vllm.distributed.kv_transfer.kv_connector.v1.nixl.stats import ( - NixlKVConnectorStats, - NixlPromMetrics, -) -from vllm.forward_context import ForwardContext -from vllm.logger import init_logger -from vllm.v1.attention.backend import AttentionMetadata -from vllm.v1.core.sched.output import SchedulerOutput -from vllm.v1.metrics.cache_hit_source import CacheHitSource -from vllm.v1.outputs import KVConnectorOutput - -if TYPE_CHECKING: - from vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_scheduler import ( - NixlBaseConnectorScheduler, - ) - from vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker import ( - NixlBaseConnectorWorker, - ) - from vllm.v1.core.kv_cache_manager import KVCacheBlocks - from vllm.v1.kv_cache_interface import KVCacheConfig - from vllm.v1.request import Request - -logger = init_logger(__name__) - - -class NixlBaseConnector(KVConnectorBase_V1, SupportsHMA): - """Base connector with common logic shared by pull and push modes.""" - - _cache_hit_source = CacheHitSource.P2P - - @property - def supports_divergent_local_hybrid_hits(self) -> bool: - return True - - def __init__( - self, - vllm_config: VllmConfig, - role: KVConnectorRole, - kv_cache_config: "KVCacheConfig", - ): - super().__init__(vllm_config, role, kv_cache_config) - assert vllm_config.kv_transfer_config is not None - assert vllm_config.kv_transfer_config.engine_id is not None - - if vllm_config.kv_transfer_config.kv_role == "kv_both": - logger.warning_once( - "Using kv_role='kv_both' with NixlConnector is deprecated " - "and will be removed in a future release. Please set " - "kv_role='kv_producer' for prefill instances and " - "kv_role='kv_consumer' for decode instances. " - ) - - parallel_config = vllm_config.parallel_config - pcp_size = parallel_config.prefill_context_parallel_size - kv_role = vllm_config.kv_transfer_config.kv_role - if pcp_size > 1 and kv_role in ("kv_consumer", "kv_both"): - raise NotImplementedError( - "NixlConnector PCP currently supports kv_producer only. " - "Consumers and kv_both require " - "prefill_context_parallel_size=1." - ) - dcp_size = parallel_config.decode_context_parallel_size - if ( - pcp_size > 1 - and dcp_size > 1 - and (parallel_config.tensor_parallel_size != 1 or dcp_size != pcp_size) - ): - raise NotImplementedError( - "NixlConnector PCP+DCP requires TP1 with DCP spanning " - "the full PCP group." - ) - # TODO: Support PCP with bidirectional KV transfer by tracking separate - # send and receive completion counts. - if pcp_size > 1 and vllm_config.kv_transfer_config.get_from_extra_config( - "bidirectional_kv_xfer", False - ): - raise NotImplementedError( - "NixlConnector PCP producers do not support bidirectional KV transfer." - ) - - self.kv_cache_config = kv_cache_config - self.engine_id: EngineId = vllm_config.kv_transfer_config.engine_id - self.kv_transfer_config = vllm_config.kv_transfer_config - # Subclasses must set self.connector_scheduler and self.connector_worker - self.connector_scheduler: NixlBaseConnectorScheduler | None = None - self.connector_worker: NixlBaseConnectorWorker | None = None - - ############################################################ - # Class Methods - ############################################################ - @classmethod - def supports_sleep_mode(cls, kv_transfer_config: KVTransferConfig) -> bool: - return False - - @classmethod - def get_required_kvcache_layout(cls, vllm_config: VllmConfig): - if vllm_config.model_config is None: - logger.warning_once( - "Unable to detect current VLLM config. " - "Fallback to default kv cache layout." - ) - return None - use_mla = vllm_config.model_config.use_mla - if use_mla: - # return None when we have mla - # as the layout should not matter in that case, - # which fallback to the default behavior. - return None - logger.info_once( - "NixlConnector setting KV cache layout to LBHNC for " - "better xfer performance." - ) - return "LBHNC" - - ############################################################ - # Scheduler Side Methods - ############################################################ - - def get_finished_count(self) -> int | None: - return None - - def get_loaded_kv_cache_group_ids(self, request: "Request") -> tuple[int, ...]: - return self._kv_cache_config.transfer_group_ids - - def get_num_new_matched_tokens( - self, request: "Request", num_computed_tokens: int - ) -> tuple[int | None, bool]: - assert self.connector_scheduler is not None - return self.connector_scheduler.get_num_new_matched_tokens( - request, num_computed_tokens - ) - - def update_state_after_alloc( - self, request: "Request", blocks: "KVCacheBlocks", num_external_tokens: int - ): - assert self.connector_scheduler is not None - return self.connector_scheduler.update_state_after_alloc( - request, blocks, num_external_tokens - ) - - def build_connector_meta( - self, - scheduler_output: SchedulerOutput, - ) -> KVConnectorMetadata: - assert self.connector_scheduler is not None - return self.connector_scheduler.build_connector_meta(scheduler_output) - - def on_new_request(self, request: "Request") -> None: - assert self.connector_scheduler is not None - self.connector_scheduler.on_new_request(request) - - def update_connector_output(self, connector_output: KVConnectorOutput): - assert self.connector_scheduler is not None - self.connector_scheduler.update_connector_output(connector_output) - - def request_finished( - self, - request: "Request", - block_ids: list[int], - ) -> tuple[bool, dict[str, Any] | None]: - assert self.connector_scheduler is not None - return self.connector_scheduler.request_finished(request, (block_ids,)) - - def request_finished_all_groups( - self, - request: "Request", - block_ids: tuple[list[int], ...], - ) -> tuple[bool, dict[str, Any] | None]: - assert self.connector_scheduler is not None - return self.connector_scheduler.request_finished(request, block_ids) - - def set_xfer_handshake_metadata_pp_aware( - self, metadata: dict[tuple[int, int], KVConnectorHandshakeMetadata] - ) -> None: - """Set handshake metadata keyed by (pp_rank, tp_rank) so the side - channel can serve every PP stage's agent metadata. - - Args: - metadata (dict): the handshake metadata to set. - - """ - assert self.connector_scheduler is not None - self.connector_scheduler.set_xfer_handshake_metadata(metadata) - - ############################################################ - # Worker Side Methods - ############################################################ - def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]): - assert self.connector_worker is not None - self.connector_worker.register_kv_caches(kv_caches) - - def set_host_xfer_buffer_ops(self, copy_operation: CopyBlocksOp): - assert self.connector_worker is not None - self.connector_worker.set_host_xfer_buffer_ops(copy_operation) - - def get_finished(self, finished_req_ids: set[str]) -> tuple[set[str], set[str]]: - """Get the finished recving and sending requests.""" - assert self.connector_worker is not None - return self.connector_worker.get_finished() - - def get_transfer_results( - self, finished_req_ids: set[str] - ) -> KVConnectorTransferResults: - assert self.connector_worker is not None - results = self.connector_worker.get_transfer_results() - if ( - self.kv_transfer_config.kv_role == "kv_producer" - and self.connector_worker.pcp_rank > 0 - and not self.connector_worker.pcp_dcp_sharded - ): - results.finished_sending.clear() - return results - - def get_block_ids_with_load_errors(self) -> set[int]: - """Get block IDs that failed to load via NIXL.""" - assert self.connector_worker is not None - return self.connector_worker.get_block_ids_with_load_errors() - - def get_kv_connector_stats(self) -> KVConnectorStats | None: - if self.connector_worker is None: - return None - return self.connector_worker.get_kv_connector_stats() - - @classmethod - def build_kv_connector_stats( - cls, data: dict[str, Any] | None = None - ) -> KVConnectorStats | None: - return ( - NixlKVConnectorStats(data=data) - if data is not None - else NixlKVConnectorStats() - ) - - @classmethod - def build_prom_metrics( - cls, - vllm_config: VllmConfig, - metric_types: dict[type[PromMetric], type[PromMetricT]], - labelnames: list[str], - per_engine_labelvalues: dict[int, list[object]], - ) -> KVConnectorPromMetrics: - return NixlPromMetrics( - vllm_config, metric_types, labelnames, per_engine_labelvalues - ) - - def wait_for_layer_load(self, layer_name: str) -> None: - """NixlConnector does not do layerwise saving.""" - pass - - def save_kv_layer( - self, - layer_name: str, - kv_layer: torch.Tensor, - attn_metadata: AttentionMetadata, - **kwargs, - ) -> None: - """NixlConnector does not save explicitly.""" - pass - - def wait_for_save(self): - assert self.connector_worker is not None - assert isinstance(self._connector_metadata, NixlConnectorMetadata) - if self.connector_worker.use_host_buffer and self.connector_worker.copy_blocks: - self.connector_worker.save_kv_to_host(self._connector_metadata) - - def has_pending_push_work(self) -> bool: - if self.connector_scheduler is not None: - return self.connector_scheduler.has_pending_push_work() - return False - - def shutdown(self): - if self.connector_worker is not None: - self.connector_worker.shutdown() - if self.connector_scheduler is not None: - self.connector_scheduler.shutdown() - - def get_handshake_metadata(self) -> KVConnectorHandshakeMetadata | None: - """Get the KVConnector handshake metadata for this connector. - This metadata is used for out-of-band connector handshake - between P/D workers. - - Returns: - KVConnectorHandshakeMetadata: the handshake metadata. - None if no handshake metadata is available. - - """ - assert self.connector_worker is not None - if ( - self.kv_transfer_config.kv_role == "kv_producer" - and self.connector_worker.pcp_rank > 0 - and not self.connector_worker.pcp_dcp_sharded - ): - return None - return self.connector_worker.xfer_handshake_metadata - - -class NixlPullConnector(NixlBaseConnector): - """Pull-based (READ) NIXL KV transfer connector.""" - - def __init__( - self, - vllm_config: VllmConfig, - role: KVConnectorRole, - kv_cache_config: "KVCacheConfig", - ): - super().__init__(vllm_config, role, kv_cache_config) - if role == KVConnectorRole.SCHEDULER: - self.connector_scheduler = NixlPullConnectorScheduler( - vllm_config, self.engine_id, kv_cache_config - ) - self.connector_worker = None - elif role == KVConnectorRole.WORKER: - self.connector_scheduler = None - self.connector_worker = NixlPullConnectorWorker( - vllm_config, self.engine_id, kv_cache_config - ) - - def start_load_kv(self, forward_context: "ForwardContext", **kwargs) -> None: - assert self.connector_worker is not None - assert isinstance(self.connector_worker, NixlPullConnectorWorker) - assert isinstance(self._connector_metadata, NixlConnectorMetadata) - self.connector_worker.start_load_kv(self._connector_metadata) - - -class NixlPushConnector(NixlBaseConnector): - """Push-based (WRITE) NIXL KV transfer connector.""" - - def __init__( - self, - vllm_config: VllmConfig, - role: KVConnectorRole, - kv_cache_config: "KVCacheConfig", - ): - super().__init__(vllm_config, role, kv_cache_config) - if vllm_config.parallel_config.decode_context_parallel_size > 1: - raise ValueError( - "NixlPushConnector does not support decode_context_parallel_size > 1." - ) - self.connector_scheduler: NixlPushConnectorScheduler | None = None - self.connector_worker: NixlPushConnectorWorker | None = None - if role == KVConnectorRole.SCHEDULER: - self.connector_scheduler = NixlPushConnectorScheduler( - vllm_config, self.engine_id, kv_cache_config - ) - elif role == KVConnectorRole.WORKER: - self.connector_worker = NixlPushConnectorWorker( - vllm_config, self.engine_id, kv_cache_config - ) - else: - raise ValueError(f"Unsupported KVConnectorRole: {role}") - - def start_load_kv(self, forward_context: "ForwardContext", **kwargs) -> None: - """Drive push processing on the worker. - - The worker enqueues registrations / finished blocks for the - background ``nixl-push-writer`` thread; the writer issues the - WRITE transfers and polls NIXL notifs without further - engine-thread involvement. - """ - assert self.connector_worker is not None - assert isinstance(self._connector_metadata, NixlConnectorMetadata) - self.connector_worker.start_load_kv(self._connector_metadata) - - -# Backward compatibility: NixlConnector is the pull-based connector. -NixlConnector = NixlPullConnector - - -__all__ = [ - "NixlBaseConnector", - "NixlConnector", - "NixlPullConnector", - "NixlPushConnector", -] diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/metadata.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/metadata.py deleted file mode 100644 index a33f620a0a..0000000000 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/metadata.py +++ /dev/null @@ -1,340 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Metadata dataclasses and helpers for the NIXL connector.""" - -from dataclasses import dataclass, field -from typing import Any - -from vllm.config import VllmConfig -from vllm.distributed.kv_transfer.kv_connector.utils import BlockIds, EngineId -from vllm.distributed.kv_transfer.kv_connector.v1.base import ( - KVConnectorHandshakeMetadata, - KVConnectorMetadata, -) -from vllm.logger import init_logger - -logger = init_logger(__name__) - -TransferHandle = int -ReqId = str - -GET_META_MSG = b"get_meta_msg" - -# Push-mode (WRITE-based) registration notification. -# Sent worker-to-worker over NIXL: D worker -> P worker, encoded as -# PUSH_REG_NOTIF_PREFIX + msgpack(registration_data). -PUSH_REG_NOTIF_PREFIX = b"PUSH_REG:" -# -# NIXL Connector Version -# -# Increment this version whenever there is an incompatible change to: -# - NixlAgentMetadata schema -# - kv_transfer_params schema or semantics -# - NIXL transfer protocol or wire format -# - KV cache memory layout or block organization -# - Any other change that breaks P/D interoperability -# -# Version History: -# 1: Initial version with compatibility checking -# 2: Add remote_request_id to kv_transfer_params -# 3: Add physical_blocks_per_logical_kv_block to NixlAgentMetadata -# 4: Add KV block lease renewal through heartbeats -# 5: Add remote_blocks_expiry_time to kv_transfer_params + handshake -# clock-sync timestamp -# 6: Validate EAGLE/MTP speculative configuration compatibility -# 7: Include NIXL transfer mode (push vs pull) in the compatibility hash -# 8: Add dcp_size and pcp_size to NixlAgentMetadata -# 9: Add block_strides -# 10: Add dense virtual transfer pages for compressed MLA caches -# 11: Add per-region transfer geometry and memory types to NixlAgentMetadata -# 12: Add per-region member names for PP push -# 13: Add packed-member layouts and order-independent packed-push backend hashes -# 14: Add Qwen3.8-Flash-Next PLE page length - #59997 -# -NIXL_CONNECTOR_VERSION: int = 14 - - -@dataclass -class NixlAgentMetadata: - engine_id: str - agent_metadata: bytes - kv_caches_base_addr: list[int] - device_id: int - num_blocks: int - block_lens: list[int] - block_strides: list[int] - kv_cache_layout: str - block_size: int - ssm_sizes: tuple[int, int] - attn_backend_name: str - physical_blocks_per_logical_kv_block: int - region_num_blocks: list[int] | None = None - region_group_ids: list[int] | None = None - region_names: list[str] | None = None - region_mem_types: list[str] | None = None - dcp_size: int = 1 - pcp_size: int = 1 - # Page length of the PLE short-conv state (None when the model has none). - ple_block_len: int | None = None - # Layer names sharing each advertised region, in region order. - region_members: list[list[str]] = field(default_factory=list) - # Packed member -> (byte offset in its region's block, bytes per page). - packed_member_layouts: dict[str, tuple[int, int]] = field(default_factory=dict) - - -@dataclass -class NixlHandshakePayload(KVConnectorHandshakeMetadata): - """Wrapper for NIXL handshake sent over the wire. - - Enables two-phase decoding for graceful compatibility checking: - 1. Decode NixlHandshakePayload to get compatibility_hash - 2. Compute local hash and compare - 3. Only if hashes match, decode agent_metadata_bytes - - This prevents decoder errors when NixlAgentMetadata schema is - incompatible, allowing graceful failure with clear error message. - """ - - compatibility_hash: str - agent_metadata_bytes: bytes # NixlAgentMetadata encoded - - -def _get_speculative_compatibility_factors( - vllm_config: VllmConfig, -) -> dict[str, Any] | None: - """Return NIXL compatibility factors for hidden-state-based speculators.""" - speculative_config = vllm_config.speculative_config - if speculative_config is None or not speculative_config.use_eagle(): - return None - - draft_model_config = speculative_config.draft_model_config - assert draft_model_config is not None - auxiliary_layer_ids = getattr( - draft_model_config.hf_config, - "eagle_aux_hidden_state_layer_ids", - None, - ) - - # kv_cache_dtype is a user override that defaults to None, meaning "inherit - # the target's --kv-cache-dtype". Resolve it to the effective value so an - # explicit setting on one side and inheritance on the other (same effective - # dtype) don't spuriously mismatch. - kv_cache_dtype = ( - speculative_config.kv_cache_dtype or vllm_config.cache_config.cache_dtype - ) - - # Note: the draft attention_backend is intentionally not hashed. Its only - # transfer-relevant effect is the KV block layout/size, which is validated - # per region at runtime in _validate_remote_agent_handshake. The connector - # only sees the raw override here (usually None = auto-select), never the - # resolved backend, so hashing it would cause false mismatches without - # catching anything the runtime layout check misses. - return { - "method": speculative_config.method, - "model": draft_model_config.model, - "revision": draft_model_config.revision, - "code_revision": draft_model_config.code_revision, - "parallel_drafting": speculative_config.parallel_drafting, - "kv_cache_dtype": str(kv_cache_dtype), - "auxiliary_layer_ids": ( - tuple(auxiliary_layer_ids) if auxiliary_layer_ids is not None else None - ), - } - - -def compute_nixl_compatibility_hash( - vllm_config: VllmConfig, - attn_backend_name: str, - transfer_mode: str = "pull", -) -> str: - """Compute compatibility hash for NIXL KV transfer. - - Hash only the factors that affect whether two NIXL instances can - successfully transfer KV cache data. - - Factors included: - - vLLM version and NIXL connector version - - Model architecture (name, dtype, KV heads, layers) - - KV cache format (dtype, sliding window) - - Attention backend - - EAGLE/MTP configuration that affects transferred state - - Transfer mode (push vs pull) - - The transfer mode is included because the push (WRITE) and pull (READ) - connectors use incompatible transfer protocols; a push connector and a - pull connector must never complete a handshake with each other. - - Note: Factors like tensor_parallel_size, block_size, and kv_cache_layout - are validated at runtime in _validate_remote_agent_handshake and are not - included in this hash to support heterogeneous deployments. - - Note - the set of factors are likely to evolve significantly over - time to be more or less permissive. - - Returns: - SHA-256 hex digest - - """ - from vllm import __version__ as vllm_version - from vllm.config.utils import hash_factors - - model_config = vllm_config.model_config - cache_config = vllm_config.cache_config - is_hma_enabled = not vllm_config.scheduler_config.disable_hybrid_kv_cache_manager - - factors = { - # Version compatibility - "vllm_version": vllm_version, - "nixl_connector_version": NIXL_CONNECTOR_VERSION, - # Model architecture - affects KV cache shape - "model": model_config.model, - "dtype": str(model_config.dtype), - "num_kv_heads": model_config.get_total_num_kv_heads(), - "head_size": model_config.get_head_size(), - "num_hidden_layers": model_config.get_total_num_hidden_layers(), - # Attention backend and KV cache dtype affect memory layout - "attn_backend_name": attn_backend_name, - "cache_dtype": str(cache_config.cache_dtype), - "is_hma_enabled": is_hma_enabled, - "speculative_config": _get_speculative_compatibility_factors(vllm_config), - # push (WRITE) and pull (READ) connectors are protocol-incompatible - "transfer_mode": transfer_mode, - } - - compat_hash = hash_factors(factors) - logger.debug( - "NIXL compatibility hash: %s (model=%s, dtype=%s, num_kv_heads=%d, " - "cache_dtype=%s, attn_backend=%s)", - compat_hash, - factors["model"], - factors["dtype"], - factors["num_kv_heads"], - factors["cache_dtype"], - attn_backend_name, - ) - return compat_hash - - -@dataclass -class HeartbeatInfo: - """Heartbeat data for a single remote engine, sent from D worker to P.""" - - req_ids: set[ReqId] - host: str - port: int - tp_size: int - dcp_size: int = 1 - pp_size: int = 1 - - -@dataclass -class RemoteMeta: - block_ids: BlockIds - host: str - port: int - engine_id: str - request_id: str - blocks_expiry_time: float | None = None - num_tokens: int | None = None - - -@dataclass -class ReqMeta: - local_block_ids: BlockIds - # To be used when logical block size does not match the kernel block size - local_physical_block_ids: BlockIds - tp_size: int - dcp_size: int = 1 - # Per-KV-cache-group logical blocks this rank already holds, i.e. its - # prefix-cache hit. Fixes where this rank's DCP slice starts relative to - # the remote's; kept per-group since hybrid models (e.g. SWA+FA) can have - # different cache-hit counts per group. - local_num_computed_blocks: tuple[int, ...] = () - remote: RemoteMeta | None = None - # Remote block size, discovered during NIXL handshake (push mode). - remote_block_size: int | None = None - # Remote producer pipeline-parallel size (push mode, D side). - pp_size: int = 1 - # True only when the scheduler parked the request in WAITING_FOR_REMOTE_KVS - # and expects it in finished_recving; notify-only recvs must not be reported. - awaiting_kvs: bool = False - # Worker-only, per-region physical pages to zero after a successful pull. - # None selects group-based completion; empty lists mean no zeroing. - region_blocks_to_zero: BlockIds | None = None - - -class NixlConnectorMetadata(KVConnectorMetadata): - def __init__(self): - self.reqs_to_recv: dict[ReqId, ReqMeta] = {} - self.reqs_to_save: dict[ReqId, ReqMeta] = {} - self.reqs_to_send: dict[ReqId, float] = {} - # The scheduler process's time.perf_counter() when this metadata was - # built. reqs_to_send deadlines are stamped with the scheduler's - # clock, which is NOT comparable across processes (perf_counter is - # process/boot-local): workers must rebase the remaining TTL onto - # their own clock via this reference. 0.0 = unset (legacy metadata). - self.scheduler_clock: float = 0.0 - self.reqs_in_batch: set[ReqId] = set() - self.reqs_not_processed: set[ReqId] = set() - # Heartbeat data grouped by remote engine, sent by D worker to P. - self.heartbeat_by_engine: dict[EngineId, HeartbeatInfo] = {} - # Push mode (D side): registration data the D worker should send to - # P workers via NIXL notification on this step. - self.push_registrations: dict[ReqId, dict[str, Any]] = {} - # Push mode (P side): newly finished request blocks to be matched - # against pending D registrations on the P worker. - self.push_finished_blocks: dict[ReqId, BlockIds] = {} - - def _add_new_req( - self, - local_block_ids: BlockIds, - kv_transfer_params: dict[str, Any], - local_num_computed_blocks: tuple[int, ...] = (), - awaiting_kvs: bool = False, - ) -> ReqMeta: - return ReqMeta( - local_block_ids=local_block_ids, - local_physical_block_ids=local_block_ids, - # P workers don't need to receive these from proxy here. - tp_size=kv_transfer_params.get("tp_size", 1), - dcp_size=kv_transfer_params.get("dcp_size", 1), - remote_block_size=kv_transfer_params.get("remote_block_size"), - pp_size=kv_transfer_params.get("pp_size", 1), - local_num_computed_blocks=local_num_computed_blocks, - awaiting_kvs=awaiting_kvs, - ) - - def add_new_req_to_save( - self, - request_id: ReqId, - local_block_ids: BlockIds, - kv_transfer_params: dict[str, Any], - ): - self.reqs_to_save[request_id] = self._add_new_req( - local_block_ids, kv_transfer_params - ) - - def add_new_req_to_recv( - self, - request_id: ReqId, - local_block_ids: BlockIds, - kv_transfer_params: dict[str, Any], - local_num_computed_blocks: tuple[int, ...] = (), - awaiting_kvs: bool = False, - ): - req = self._add_new_req( - local_block_ids, - kv_transfer_params, - local_num_computed_blocks, - awaiting_kvs, - ) - req.remote = RemoteMeta( - block_ids=kv_transfer_params["remote_block_ids"], - engine_id=kv_transfer_params["remote_engine_id"], - request_id=kv_transfer_params["remote_request_id"], - host=kv_transfer_params["remote_host"], - port=kv_transfer_params["remote_port"], - blocks_expiry_time=kv_transfer_params.get("remote_blocks_expiry_time"), - num_tokens=kv_transfer_params.get("remote_num_tokens"), - ) - self.reqs_to_recv[request_id] = req diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/pull_scheduler.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/pull_scheduler.py deleted file mode 100644 index f283a40d3c..0000000000 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/pull_scheduler.py +++ /dev/null @@ -1,292 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Pull-specific scheduler-side logic for the NIXL connector.""" - -import time -from typing import TYPE_CHECKING, Any - -from vllm.distributed.kv_transfer.kv_connector.utils import BlockIds -from vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_scheduler import ( - NixlBaseConnectorScheduler, -) -from vllm.logger import init_logger - -if TYPE_CHECKING: - from vllm.config import VllmConfig - from vllm.v1.core.kv_cache_manager import KVCacheBlocks - from vllm.v1.kv_cache_interface import KVCacheConfig - from vllm.v1.request import Request - -logger = init_logger(__name__) - - -class NixlPullConnectorScheduler(NixlBaseConnectorScheduler): - """Pull-specific scheduler logic (READ-based KV transfer).""" - - def __init__( - self, - vllm_config: "VllmConfig", - engine_id: str, - kv_cache_config: "KVCacheConfig", - ): - super().__init__(vllm_config, engine_id, kv_cache_config) - - def get_num_new_matched_tokens( - self, request: "Request", num_computed_tokens: int - ) -> tuple[int, bool]: - """For remote prefill, pull all prompt blocks from remote - asynchronously relative to engine execution. - - Args: - request (Request): the request object. - num_computed_tokens (int): the number of locally - computed tokens for this request - Returns: - * the number of tokens that can be loaded from the - external KV cache beyond what is already computed. - * true if the external KV cache tokens will be loaded - asynchronously (between scheduler steps). - - """ - params = request.kv_transfer_params - logger.debug( - "NIXLConnector get_num_new_matched_tokens: " - "num_computed_tokens=%s, kv_transfer_params=%s", - num_computed_tokens, - params, - ) - - if params is not None and params.get("do_remote_prefill"): - # Remote prefill: get all prompt blocks from remote. - token_ids = request.prompt_token_ids or [] - actual = self._get_remote_prefill_token_count(len(token_ids)) - count = actual - num_computed_tokens - if count > 0: - return count, True - - if ( - params is not None - and params.get("do_remote_decode") - and params.get("remote_block_ids") - and all( - p in params - for p in ( - "remote_engine_id", - "remote_request_id", - "remote_host", - "remote_port", - ) - ) - ): - # Decode node has kv blocks for part of prefill request, so, provide them - # as an external token count to scheduler. - # The tokens will be loaded if not already present - # in the prefill node local cache - remote_num_tokens = params.get("remote_num_tokens") or 0 - count = ( - min(remote_num_tokens, request.num_prompt_tokens) - num_computed_tokens - ) - if count > 0: - # Check kv_recompute_threshold: skip pull if - # remote tokens are below the threshold. - if ( - self.kv_recompute_threshold > 0 - and count < self.kv_recompute_threshold - ): - logger.debug( - "Skipping remote pull for %s: %d remote tokens < threshold %d", - request.request_id, - count, - self.kv_recompute_threshold, - ) - return 0, False - return count, True - - # No remote prefill for this request. - return 0, False - - def update_state_after_alloc( - self, request: "Request", blocks: "KVCacheBlocks", num_external_tokens: int - ): - params = request.kv_transfer_params - logger.debug( - "NIXLConnector update_state_after_alloc: " - "num_external_tokens=%s, kv_transfer_params=%s", - num_external_tokens, - params, - ) - - if not params: - return - - if params.get("do_remote_decode") or ( - params.get("do_remote_prefill") and self.is_bidirectional_kv_xfer_enabled - ): - self._reqs_in_batch.add(request.request_id) - if self.use_host_buffer and params.get("do_remote_decode"): - # NOTE: when accelerator is not directly supported by Nixl, - # prefilled blocks need to be saved to host memory before transfer. - self._reqs_need_save[request.request_id] = request - elif params.get("do_remote_prefill") or ( - params.get("do_remote_decode") - and self.is_bidirectional_kv_xfer_enabled - and not params.get("_remote_blocks_processed") - ): - if params.get("remote_block_ids"): - if all( - p in params - for p in ( - "remote_engine_id", - "remote_request_id", - "remote_host", - "remote_port", - ) - ): - # If remote_blocks and num_external_tokens = 0, we have - # a full prefix cache hit on the local node. We need to call - # send_notif in _read_blocks to free the memory on the remote node. - - unhashed_local_block_ids: BlockIds = ( - blocks.get_unhashed_block_ids_all_groups() - if num_external_tokens > 0 - else () - ) - local_block_ids = self.get_exchange_clipped_blocks( - unhashed_local_block_ids - ) - # Blocks covered by the local prefix cache, per KV cache group. - # Each count fixes where that group's DCP slice starts, which the - # worker needs to line up with the remote's slice. - local_num_computed_blocks = tuple( - sum( - block.block_hash is not None and not block.is_null - for block in group - ) - for group in blocks.blocks - ) - - # Get unhashed blocks to pull from remote. Mind that a full prefix - # cache hit is indicated with an empty list. - self._reqs_need_recv[request.request_id] = ( - request, - local_block_ids, - local_num_computed_blocks, - # Parked in WAITING_FOR_REMOTE_KVS only when there is - # something to pull; a full local hit stays RUNNING. - num_external_tokens > 0, - ) - - else: - logger.warning( - "Got invalid KVTransferParams: %s. This " - "request will not utilize KVTransfer", - params, - ) - else: - assert num_external_tokens == 0 - # Only trigger 1 KV transfer per request. - params["do_remote_prefill"] = False - params["_remote_blocks_processed"] = True - - def request_finished( - self, - request: "Request", - block_ids: "BlockIds", - ) -> tuple[bool, dict[str, Any] | None]: - """Once a request is finished, determine whether request blocks - should be freed now or will be sent asynchronously and freed later. - """ - from vllm.v1.request import RequestStatus - - params = request.kv_transfer_params - logger.debug( - "NIXLConnector request_finished(%s), request_status=%s, " - "kv_transfer_params=%s", - request.request_id, - request.status, - params, - ) - if not params: - return False, None - - is_p_node = bool(params.get("do_remote_decode")) - is_d_node = not is_p_node - - # Stop heartbeating for aborted requests that never reached finished_recving: - # normal path cleans up in update_connector_output. - self._stop_heartbeat(request.request_id) - - if params.get("do_remote_prefill"): - # If do_remote_prefill is still True when the request is finished, - # update_state_after_alloc must not have been called (the request - # must have been aborted before it was scheduled, e.g. via the - # abort_immediately path used to clean up KV-transfer requests - # rejected at the D-side serving layer). - # To avoid stranding the prefill blocks in the prefill instance, - # we must add empty block_ids to _reqs_need_recv so that our - # worker side will notify and free blocks in the prefill instance. - self._reqs_need_recv[request.request_id] = (request, [], (), False) - params["do_remote_prefill"] = False - return False, None - - if is_d_node and not self.is_bidirectional_kv_xfer_enabled: - return False, None - - if request.status not in ( - RequestStatus.FINISHED_LENGTH_CAPPED, - RequestStatus.FINISHED_STOPPED, - ): - # Also include the case of a P/D Prefill request with immediate - # block free (eg abort). Stop tracking this request. - self._reqs_not_processed.add(request.request_id) - # Clear _reqs_need_save if a request is aborted as partial prefill. - self._reqs_need_save.pop(request.request_id, None) - return False, None - - # TODO: check whether block_ids actually ever be 0. If not we could - # remove the conditional below - delay_free_blocks = any(len(group) > 0 for group in block_ids) - remote_num_tokens = 0 - blocks_expiry_time = None - if delay_free_blocks: - # Prefill request on remote. It will be read from D upon completion - request_kv_blocks_ttl = self._kv_lease_duration - if is_d_node: - # For blocks pinned on D, use a simpler timeout for now instead of a - # lease mechanism as turn2 request is client-driven. - request_kv_blocks_ttl = self.decoder_kv_blocks_ttl - logger.debug( - "NIXLConnector request_finished(%s) waiting for %d seconds " - "before releasing blocks", - request.request_id, - request_kv_blocks_ttl, - ) - self._reqs_need_send[request.request_id] = ( - time.perf_counter() + request_kv_blocks_ttl - ) - if is_d_node: - # D blocks expiry time exported for the turn-2 readback. - blocks_expiry_time = self._reqs_need_send[request.request_id] - # NOTE HMA will "mark" empty/null blocks in groups with 0s (eg SWA ones), - # trimming down after allocating for the whole sequence length. Empty - # blocks are always at the start of the list. - # Here we "unpad" blocks to send the actual remote blocks to be read. - block_ids = self.get_exchange_clipped_blocks(block_ids) - - remote_num_tokens = request.num_computed_tokens - - return delay_free_blocks, dict( - do_remote_prefill=is_p_node, - do_remote_decode=is_d_node, - remote_block_ids=block_ids, - remote_engine_id=self.engine_id, - remote_request_id=request.request_id, - remote_host=self.side_channel_host, - remote_port=self.side_channel_port, - tp_size=self.transfer_tp_size, - dcp_size=self.vllm_config.parallel_config.decode_context_parallel_size, - pp_size=self.vllm_config.parallel_config.pipeline_parallel_size, - remote_num_tokens=remote_num_tokens, - remote_blocks_expiry_time=blocks_expiry_time, - transfer_mode=self._TRANSFER_MODE, - ) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/pull_worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/pull_worker.py deleted file mode 100644 index 16a25149ea..0000000000 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/pull_worker.py +++ /dev/null @@ -1,693 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Pull-specific (READ) worker-side logic for the NIXL connector.""" - -import time -from typing import TYPE_CHECKING - -import numpy as np - -from vllm.distributed.kv_transfer.kv_connector.utils import BlockIds -from vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker import ( - NixlBaseConnectorWorker, -) -from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import ( - NixlConnectorMetadata, - ReqMeta, -) -from vllm.distributed.kv_transfer.kv_connector.v1.nixl.tp_mapping import ( - ReadSpec, - _is_attention_spec, -) -from vllm.logger import init_logger -from vllm.utils.math_utils import cdiv - -if TYPE_CHECKING: - from vllm.config import VllmConfig - from vllm.v1.kv_cache_interface import KVCacheConfig - -logger = init_logger(__name__) - -# Slack (seconds) subtracted from D's exported block-expiry deadline on the turn-2 -# readback, absorbing clock-offset error and read latency. -_KV_BLOCKS_EXPIRY_SAFETY_MARGIN = 5.0 - - -class NixlPullConnectorWorker(NixlBaseConnectorWorker): - """Pull-specific (READ) worker logic.""" - - def __init__( - self, - vllm_config: "VllmConfig", - engine_id: str, - kv_cache_config: "KVCacheConfig", - ): - super().__init__(vllm_config, engine_id, kv_cache_config) - - def start_load_kv(self, metadata: NixlConnectorMetadata): - """Start loading by triggering non-blocking nixl_xfer. - We check for these trnxs to complete in each step(). - """ - for req_id, meta in metadata.reqs_to_recv.items(): - meta.local_physical_block_ids = self._logical_to_kernel_block_ids( - meta.local_block_ids, self._physical_blocks_per_logical_kv_block - ) - assert meta.remote is not None - # Remote block IDs are kept logical here; expanded in - # _read_blocks_for_req using the remote engine's phys ratio. - remote_engine_id = meta.remote.engine_id - logger.debug( - "start_load_kv for request %s from remote engine %s. " - "Num local_block_ids: %s. Num remote_block_ids: %s. ", - req_id, - remote_engine_id, - len(meta.local_physical_block_ids), - len(meta.remote.block_ids), - ) - # Full local hits and aborted cleanup only notify P; no recv is awaited. - # On a full local hit: - # - Notification failure must not fail the request: its KV is local. - # - Receive completion must not be reported: num_external_tokens == 0, - # so the scheduler never entered WAITING_FOR_REMOTE_KVS to begin with. - # Aborted cleanup requests have already been removed from the scheduler. - if meta.awaiting_kvs or any(meta.local_block_ids): - self._recving_metadata[req_id] = meta - self._read_blocks_for_req(req_id, meta) - - # Start transfers for requests whose handshakes have now finished. - while not self._ready_requests.empty(): - self._read_blocks_for_req(*self._ready_requests.get_nowait()) - - if self.pcp_rank > 0 and not self.pcp_dcp_sharded: - # Replicated-KV PCP: only PCP rank 0 serves the KV, so this rank - # has nothing to send. Report the requests as sent right away so - # the scheduler-side aggregation (world_size workers, and any - # sibling connector inside a MultiConnector) still completes. - self._replicated_pcp_done_sending.update(metadata.reqs_to_send) - return - - # Keep around the requests that have been part of a batch. This is - # needed because async scheduling pushes the misalignment between the - # moment in which requests expiration is set (P side) and the moment in - # which blocks are read from D. As P can now more easily lag behind D - # while processing the next batch, we make sure to only set an - # expiration for requests that have not been read from D yet. - for req_id in metadata.reqs_in_batch: - self._reqs_to_process.add(req_id) - - # Remove all requests that are not to be processed (eg aborted). - for req_id in metadata.reqs_not_processed: - self._reqs_to_process.discard(req_id) - # We should never get an abort after setting an expiry timer - assert req_id not in self._reqs_to_send - - # Add to requests that are waiting to be read and track expiration. - # Deadlines are stamped with the scheduler process's perf_counter, - # which is not comparable to ours when the worker runs in another - # process on another node (perf_counter epochs differ by boot time). - # Rebase the remaining TTL onto our clock; broadcast latency only - # lengthens the lease, which is the safe direction. A cross-node - # epoch gap larger than the TTL otherwise expires the lease on - # arrival and the blocks are freed before D reads them. - now_local = time.perf_counter() - for req_id, expiration_time in metadata.reqs_to_send.items(): - if req_id in self._reqs_to_process: - if metadata.scheduler_clock: - expiration_time = now_local + ( - expiration_time - metadata.scheduler_clock - ) - self._reqs_to_send[req_id] = expiration_time - - # Send heartbeats to P-side engines to keep KV blocks alive while - # requests sit in the D scheduler WAITING queue. - self._send_heartbeats(metadata) - - def _is_turn2_read_expired(self, meta: ReqMeta) -> bool: - """Whether D's cached blocks for this turn-2 readback have (nearly) expired.""" - assert meta.remote is not None - blocks_expiry_time = meta.remote.blocks_expiry_time - # Deadline may be absent (router may not forward it) -> read as usual. - if blocks_expiry_time is None or not meta.local_physical_block_ids: - return False - clock_offset = self._engine_clock_offset[meta.remote.engine_id] - deadline = blocks_expiry_time - clock_offset - return time.perf_counter() + _KV_BLOCKS_EXPIRY_SAFETY_MARGIN >= deadline - - def _read_blocks_for_req(self, req_id: str, meta: ReqMeta): - assert meta.remote is not None and self.transfer_topo is not None - engine_id = meta.remote.engine_id - if engine_id in self._invalid_remote_engines: - self._handle_failed_transfer( - req_id, None, self._recv_failures, record_failed_transfer=False - ) - return - if engine_id not in self._remote_agents: - # Also re-handshakes queued requests that outlived invalid-peer cleanup. - self._background_nixl_handshake(req_id, engine_id, meta) - return - # Update last activity from this remote. Mind that cleanup is done on main - # thread (this one), so we don't race on this structure. - self._engine_last_active[engine_id] = time.perf_counter() - - if self._bidirectional_kv_xfer_enabled and self._is_turn2_read_expired(meta): - logger.warning( - "Declining expired remote read for %s from engine %s.", - req_id, - engine_id, - ) - self.xfer_stats.record_kv_expired_req() - # KV expiry is reported separately from transport failures, so only - # the state cleanup side of _handle_failed_transfer runs here. - self._handle_failed_transfer( - req_id, None, self._recv_failures, record_failed_transfer=False - ) - return - - if any(len(group) > 0 for group in meta.local_block_ids): - # The scheduler waits for finished_recving from *every* worker. - # Under DCP a rank's slice can legitimately come out empty when its - # interleaved positions fall past the end of the sequence. _read_blocks - # then takes the notify-only path without registering a transfer. - # Seed the entry so this rank still reports completion. - self._recving_transfers.setdefault(req_id, []) - - plan = self.tp_mappings[engine_id] - remote_info = self.transfer_topo.get_engine_info(engine_id) - tp_ratio = self.transfer_topo.tp_ratio(remote_info.remote_tp_size) - - dcp_active = self.dcp_size > 1 or remote_info.remote_dcp_size > 1 - local_block_ids = meta.local_physical_block_ids - remote_region_groups = self.dst_region_group_ids[engine_id] - local_region_groups = self.region_group_ids or remote_region_groups - if not local_block_ids: - # Region expansion cannot index empty groups. Pass empty specs to - # _read_blocks so its existing cache-hit notification path runs. - read_specs = [ - ReadSpec(remote_rank=rank, local_block_ids=[], remote_block_ids=[]) - for rank in plan.all_source_ranks - ] - elif local_region_groups != remote_region_groups: - if not self.use_mla or self._has_mamba: - raise NotImplementedError( - "Different NIXL cache-group layouts are only supported for " - "pure MLA models" - ) - if self.block_size != remote_info.remote_block_size: - raise NotImplementedError( - "Region-mapped NIXL transfers require matching physical block sizes" - ) - remote_physical_block_ids = self._logical_to_kernel_block_ids( - meta.remote.block_ids, - remote_info.remote_physical_blocks_per_logical, - ) - remote_by_region = self._block_ids_by_region( - remote_physical_block_ids, remote_region_groups - ) - local_by_region = self._block_ids_by_region( - local_block_ids, local_region_groups - ) - num_computed_blocks = None - num_remote_blocks = None - if ( - meta.remote.num_tokens is not None - and meta.local_num_computed_blocks - and all(group >= 0 for group in local_region_groups) - and all(group >= 0 for group in remote_region_groups) - ): - transfer_groups = self.kv_cache_config.transfer_group_ids - num_computed_blocks = [ - meta.local_num_computed_blocks[transfer_groups[group]] - * self._physical_blocks_per_logical_kv_block - for group in local_region_groups - ] - num_remote_blocks = cdiv( - meta.remote.num_tokens, remote_info.remote_block_size - ) - elif any(local_by_region) and ( - dcp_active - or remote_info.remote_physical_blocks_per_logical - != self._physical_blocks_per_logical_kv_block - ): - raise NotImplementedError( - "Region-mapped pulls with DCP or different logical block sizes " - "require remote_num_tokens, per-group prefix counts " - "and unshared regions" - ) - read_specs = [ - ReadSpec( - rank, - *self._apply_prefix_caching_by_region( - local_by_region, - remote_by_region, - num_computed_blocks=num_computed_blocks, - num_remote_blocks=num_remote_blocks, - remote_rank=rank, - remote_dcp_size=remote_info.remote_dcp_size, - ), - block_ids_by_region=True, - ) - for rank in plan.all_source_ranks - ] - meta.region_blocks_to_zero = [ - list(blocks[sum(len(spec.local_block_ids[r]) for spec in read_specs) :]) - for r, blocks in enumerate(local_by_region) - ] - else: - remote_logical_block_ids = meta.remote.block_ids - meta.remote.block_ids = self._logical_to_kernel_block_ids( - remote_logical_block_ids, - remote_info.remote_physical_blocks_per_logical, - ) - num_groups = len(meta.local_block_ids) - - def group_ids(block_ids: BlockIds, rank: int) -> list[list[int]]: - return [ - list(block_ids[g]) if rank in plan.source_ranks_per_group[g] else [] - for g in range(num_groups) - ] - - read_specs = [] - for rank in plan.all_source_ranks: - if dcp_active: - local_ids = group_ids(meta.local_block_ids, rank) - remote_ids = group_ids(remote_logical_block_ids, rank) - for g in range(num_groups): - if not local_ids[g] or not _is_attention_spec( - self._group_spec_types[g] - ): - continue - local_ids[g], remote_ids[g] = self._apply_dcp_prefix_caching( - local_ids[g], - remote_ids[g], - remote_rank=rank, - local_dcp_size=self.dcp_size, - local_dcp_rank=self.dcp_rank, - remote_dcp_size=remote_info.remote_dcp_size, - local_num_computed_blocks=( - meta.local_num_computed_blocks[g] - ), - ) - local_physical_ids = self._logical_to_kernel_block_ids( - local_ids, self._physical_blocks_per_logical_kv_block - ) - remote_physical_ids = self._logical_to_kernel_block_ids( - remote_ids, - remote_info.remote_physical_blocks_per_logical, - ) - else: - local_physical_ids = group_ids(meta.local_physical_block_ids, rank) - remote_physical_ids = group_ids(meta.remote.block_ids, rank) - read_specs.append( - ReadSpec( - remote_rank=rank, - local_block_ids=local_physical_ids, - remote_block_ids=remote_physical_ids, - ) - ) - - # D may have to perform multiple reads from different remote ranks. - # Pure MLA reads once because its cache is replicated. Hybrid - # MLA+SSM still needs one read per SSM source rank. With DCP, pure - # MLA may also read from multiple ranks (disjoint token slices). - if self.use_mla and tp_ratio < 0 and not self._has_mamba and not dcp_active: - assert len(read_specs) == 1 - - for i, spec in enumerate(read_specs): - remote_block_size = remote_info.remote_block_size - logger.debug( - "Remote agent %s available, calling _read_blocks" - " on remote rank %s with remote block size %s for req %s", - meta.remote.engine_id, - spec.remote_rank, - remote_block_size, - req_id, - ) - # Get side handles. - if tp_ratio < 0 and (not self.use_mla or len(read_specs) > 1): - # Remote tp_size > local tp_size: we must perform multiple - # reads. Get the memory chunk onto which we will write to. - split_key = (tp_ratio, remote_block_size) - local_xfer_side_handle = self.src_xfer_handles_by_tp_ratio[split_key][i] - local_dram_handle = ( - self._dram_src_handles_by_tp_ratio[split_key][i] - if self._mixed_mem_types - else None - ) - else: - # Single read from remote, we write to the whole memory region. - # Also handle remote block size different from local block size. - local_xfer_side_handle = self.src_xfer_handles_by_block_size[ - remote_block_size - ] - local_dram_handle = ( - self._dram_src_handles_by_block_size[remote_block_size] - if self._mixed_mem_types - else None - ) - - # Destination handle: remote_engine_id -> remote_rank -> handle. - remote_xfer_side_handle = self.dst_xfer_side_handles[meta.remote.engine_id][ - spec.remote_rank - ] - - # Once a read routes the request to failure reporting, the - # scheduler may free and reuse its blocks, so no sibling READs - # may be posted (and P must not be notified). - if not self._read_blocks( - read_spec=spec, - request_id=req_id, - dst_engine_id=meta.remote.engine_id, - remote_request_id=meta.remote.request_id, - local_xfer_side_handle=local_xfer_side_handle, - local_dram_handle=local_dram_handle, - remote_xfer_side_handle=remote_xfer_side_handle, - expected_consumers=plan.local_consumers, - awaiting_kvs=meta.awaiting_kvs, - ): - return - - if self.use_mla and tp_ratio < 0 and len(read_specs) == 1: - # ..but we still need to notify the other remote ranks that we - # have the blocks we need so they can update the request state. - # Same thing for DCP (tp_size == dcp_size), so the raw tp_ratio already - # reflects whether any remote replica is left unchosen. - notif_id = f"{meta.remote.request_id}:{plan.local_consumers}".encode() - remote_agents = self._remote_agents[meta.remote.engine_id] - for rank_to_notify, agent in remote_agents.items(): - if rank_to_notify != (0, read_specs[0].remote_rank): - try: - self.nixl_wrapper.send_notif(agent, notif_msg=notif_id) - except Exception as e: - self._log_failure( - failure_type="notification_failed", - msg="Remote rank will not update request state. " - "This may indicate network issues.", - req_id=req_id, - error=e, - dst_engine_id=meta.remote.engine_id, - remote_rank=rank_to_notify[1], - remote_agent_name=agent, - ) - self.xfer_stats.record_failed_notification() - - def _read_blocks( - self, - read_spec: ReadSpec, - dst_engine_id: str, - request_id: str, - remote_request_id: str, - local_xfer_side_handle: int, - local_dram_handle: int | None, - remote_xfer_side_handle: int, - expected_consumers: int, - awaiting_kvs: bool, - ) -> bool: - """Post a READ point-to-point xfer request from a single local worker to - a single remote worker. - - Returns True when the read was posted (or was unnecessary), False - when the request was routed to failure reporting — the caller must - not post further transfers for it. - """ - assert self.transfer_topo is not None - remote_rank = read_spec.remote_rank - local_block_ids = read_spec.local_block_ids - remote_block_ids = read_spec.remote_block_ids - - remote_info = self.transfer_topo.get_engine_info(dst_engine_id) - block_size_ratio = self.transfer_topo.block_size_ratio( - remote_info.remote_block_size - ) - if block_size_ratio > 1: - if read_spec.block_ids_by_region: - raise NotImplementedError( - "Region-mapped NIXL transfers require matching physical block sizes" - ) - local_block_ids, remote_block_ids = ( - self._map_block_ids_for_block_size_ratio( - local_block_ids, remote_block_ids, block_size_ratio - ) - ) - # NOTE(rob): having the staging blocks be on the READER side is - # not going to work well (since we will have to call rearrange tensors). - # after we detect the txn is complete (which means we cannot make the - # read trxn async easily). If we want to make "READ" happen cleanly, - # then we will need to have the staging blocks on the remote side. - - # NOTE(rob): according to nvidia the staging blocks are used to - # saturate IB with heterogeneous TP sizes. - - # Number of local workers that will notify this producer worker. - # Propagate on notification so dst worker can wait before freeing. - notif_id = f"{remote_request_id}:{expected_consumers}".encode() - - # Full prefix cache hit: do not need to read remote blocks, - # just notify P worker that we have the blocks we need. - if not any(len(group) > 0 for group in local_block_ids): - # A full prefix cache hit is indicated with an empty list. - agent_name = self._remote_agents[dst_engine_id][(0, remote_rank)] - try: - self.nixl_wrapper.send_notif(agent_name, notif_msg=notif_id) - except Exception as e: - self._log_failure( - failure_type="notification_failed", - msg="P worker blocks will be freed after timeout. " - "This may indicate network issues.", - req_id=request_id, - error=e, - dst_engine_id=dst_engine_id, - remote_rank=remote_rank, - remote_agent_name=agent_name, - ) - self.xfer_stats.record_failed_notification() - # Report even on notification failure: the KV is already local, and - # an unreported parked request would hold its blocks forever. - # Notify-only recvs must stay unreported (scheduler asserts). - if awaiting_kvs: - self._recving_transfers.setdefault(request_id, []) - return True - - if not read_spec.block_ids_by_region: - assert ( - len(remote_block_ids) - == len(local_block_ids) - == len(self.kv_cache_config.transfer_groups) - ) - if not (self.dcp_size > 1 or remote_info.remote_dcp_size > 1): - local_block_ids, remote_block_ids = self._apply_prefix_caching( - decode_block_ids=local_block_ids, - prefill_block_ids=remote_block_ids, - decode_physical_per_logical=( - self._physical_blocks_per_logical_kv_block - ), - prefill_physical_per_logical=( - remote_info.remote_physical_blocks_per_logical - ), - ) - - # NOTE (nicolo) With homogeneous TP, each TP worker loads KV from - # corresponding rank. With heterogeneous TP, fixing D>P, the D tp - # workers will issue xfers to parts of the P worker remote kv caches. - - # Get descs ids. - remote_block_descs_ids = self._compute_desc_ids( - block_ids=remote_block_ids, - dst_num_blocks=self.dst_num_blocks[dst_engine_id], - block_size_ratio=None, - physical_blocks_per_logical=remote_info.remote_physical_blocks_per_logical, - region_num_blocks=(self.dst_region_num_blocks.get(dst_engine_id) or None), - region_group_ids=( - list(range(self.num_regions)) - if read_spec.block_ids_by_region - else (self.dst_region_group_ids.get(dst_engine_id) or None) - ), - uses_region_group_mapping=( - self.num_regions > 1 - if read_spec.block_ids_by_region - else self.dst_uses_region_group_mapping[dst_engine_id] - ), - ) - local_block_descs_ids = self._compute_desc_ids( - block_ids=local_block_ids, - dst_num_blocks=self.dst_num_blocks[self.engine_id], - block_size_ratio=block_size_ratio, - physical_blocks_per_logical=self._physical_blocks_per_logical_kv_block, - region_num_blocks=(self.dst_region_num_blocks.get(self.engine_id) or None), - region_group_ids=( - list(range(self.num_regions)) - if read_spec.block_ids_by_region - else (self.region_group_ids or None) - ), - uses_region_group_mapping=( - self.num_regions > 1 - if read_spec.block_ids_by_region - else self._uses_region_group_mapping - ), - ) - - assert len(local_block_descs_ids) == len(remote_block_descs_ids) - - # Prepare transfer with Nixl. - handle = None - try: - if self._mixed_mem_types: - self._read_blocks_mixed( - request_id=request_id, - local_block_size_key=remote_info.remote_block_size, - local_device_handle=local_xfer_side_handle, - local_dram_handle=local_dram_handle, - remote_xfer_side_handle=remote_xfer_side_handle, - local_block_descs_ids=local_block_descs_ids, - remote_block_descs_ids=remote_block_descs_ids, - notif_agent=self._remote_agents[dst_engine_id][(0, remote_rank)], - notif_id=notif_id, - ) - return True - handle = self.nixl_wrapper.make_prepped_xfer( - "READ", - local_xfer_side_handle, - local_block_descs_ids, - remote_xfer_side_handle, - remote_block_descs_ids, - notif_msg=notif_id, - ) - - # Begin async xfer. - self.nixl_wrapper.transfer(handle) - - # Use handle to check completion in future step(). - self._recving_transfers[request_id].append(handle) - return True - except Exception as e: - self._log_failure( - failure_type="transfer_setup_failed", - req_id=request_id, - msg="Deferring failure reporting until outstanding transfers finish", - error=e, - dst_engine_id=dst_engine_id, - remote_rank=remote_rank, - ) - if not self._handle_failed_transfer( - request_id, handle, self._recv_failures - ): - assert handle is not None - self._recving_transfers[request_id].append(handle) - return False - - def _read_blocks_mixed( - self, - request_id: str, - local_block_size_key: int, - local_device_handle: int, - local_dram_handle: int | None, - remote_xfer_side_handle: int, - local_block_descs_ids: np.ndarray, - remote_block_descs_ids: np.ndarray, - notif_agent: str, - notif_id: bytes, - ) -> None: - """Split a READ across the local DRAM and device descriptor lists.""" - desc_is_dram = self._desc_is_dram_by_block_size[local_block_size_key] - desc_pos = self._desc_pos_by_block_size[local_block_size_key] - local_ids = np.asarray(local_block_descs_ids) - remote_ids = np.asarray(remote_block_descs_ids) - is_dram = desc_is_dram[local_ids] - - if self._skip_dram_xfer: - # TP rank 0 reads the DRAM (shared host pool) part for every - # local rank; keep only the device part. Register the request - # even if nothing is left, so it still completes and notifies. - keep = ~is_dram - local_ids = local_ids[keep] - remote_ids = remote_ids[keep] - is_dram = is_dram[keep] - self._recving_transfers.setdefault(request_id, []) - assert local_dram_handle is not None or not is_dram.any() - reads = ( - (is_dram, local_dram_handle), - (~is_dram, local_device_handle), - ) - handles: list[int] = [] - try: - for mask, local_handle in reads: - if mask.any(): - handles.append( - self.nixl_wrapper.make_prepped_xfer( - "READ", - local_handle, - desc_pos[local_ids[mask]], - remote_xfer_side_handle, - remote_ids[mask], - ) - ) - except Exception: - for handle in handles: - if not self._try_release_xfer_handle(request_id, handle): - self._recving_transfers[request_id].append(handle) - raise - - self._pending_recv_notifs.setdefault(request_id, []).append( - (notif_agent, notif_id) - ) - for i, handle in enumerate(handles): - try: - self.nixl_wrapper.transfer(handle) - except Exception: - for unstarted in handles[i:]: - if not self._try_release_xfer_handle(request_id, unstarted): - self._recving_transfers[request_id].append(unstarted) - raise - self._recving_transfers[request_id].append(handle) - - def _get_new_notifs(self) -> set[str]: - """Get req_ids which got a remote xfer message. When multiple consumers - are reading from the same producer (heterogeneous TP or DCP - scenario), wait for all consumers to be done pulling. - - Also handles heartbeat notifications ("HB:req1,req2,...") by - extending the lease on the referenced requests. - """ - assert self.transfer_topo is not None - notified_req_ids: set[str] = set() - for notifs in self.nixl_wrapper.get_new_notifs().values(): - for notif in notifs: - msg = notif.decode("utf-8") - - # Handle heartbeat messages from D-side. - if msg.startswith("HB:"): - self._handle_heartbeat(msg[3:]) - continue - - req_id, expected_consumers = msg.rsplit(":", 1) - if ( - req_id not in self._reqs_to_send - and req_id not in self._reqs_to_process - ): - self.xfer_stats.record_notification_after_expiry() - logger.error( - "Potentially invalid KV blocks for " - "unrecognized request %s were retrieved by " - "a decode worker. They may have expired.", - req_id, - ) - continue - - # Every reader of this req_id reports the same count (it's - # derived from aggregate topology, not the specific rank), - # so repeated notifications never disagree on it. - self.expected_consumer_notifications_by_req[req_id] = int( - expected_consumers - ) - - self.consumer_notification_counts_by_req[req_id] += 1 - # Wait all consumers (D) to be done reading before freeing. - if ( - self.consumer_notification_counts_by_req[req_id] - == self.expected_consumer_notifications_by_req[req_id] - ): - notified_req_ids.add(req_id) - del self.consumer_notification_counts_by_req[req_id] - del self.expected_consumer_notifications_by_req[req_id] - self._reqs_to_process.remove(req_id) - self._reqs_to_send.pop(req_id, None) - return notified_req_ids diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/push_scheduler.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/push_scheduler.py deleted file mode 100644 index a007e0c5a3..0000000000 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/push_scheduler.py +++ /dev/null @@ -1,357 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Push-specific scheduler-side logic for the NIXL connector. - -In push mode, scheduler-side responsibilities are: - -* D side (decode): on ``update_state_after_alloc``, stash registration data - (D's identity + locally allocated block IDs) into - ``_push_pending_registrations``. The D worker drains it from - ``meta.push_registrations`` next step and sends a NIXL notification to the - P worker (no scheduler-level networking). -* P side (prefill): on ``request_finished``, stash the finished block IDs - into ``_finished_request_blocks`` for the lease, and into - ``_newly_finished_push_blocks`` so the P worker picks them up via - ``meta.push_finished_blocks`` and matches against any D registrations - it already received via NIXL notifications. -* Both sides: ``has_pending_push_work`` keeps the engine main loop stepping - while pushes are in flight. ``update_connector_output`` cleans up - ``_finished_request_blocks`` once the WRITE completes. - -A soft per-registration watchdog on the D scheduler fails requests that have -been registered but not fulfilled within a configurable timeout. -""" - -from __future__ import annotations - -import time -from typing import TYPE_CHECKING, Any - -from vllm.distributed.kv_transfer.kv_connector.utils import BlockIds -from vllm.distributed.kv_transfer.kv_connector.v1.base import ( - KVConnectorMetadata, -) -from vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_scheduler import ( - NixlBaseConnectorScheduler, -) -from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import ( - NixlConnectorMetadata, - ReqId, -) -from vllm.logger import init_logger - -if TYPE_CHECKING: - from vllm.config import VllmConfig - from vllm.v1.core.kv_cache_manager import KVCacheBlocks - from vllm.v1.core.sched.output import SchedulerOutput - from vllm.v1.kv_cache_interface import KVCacheConfig - from vllm.v1.outputs import KVConnectorOutput - from vllm.v1.request import Request - -logger = init_logger(__name__) - - -class NixlPushConnectorScheduler(NixlBaseConnectorScheduler): - """Push-specific scheduler logic (WRITE-based KV transfer). - - All P2P communication is deferred to the worker level via NIXL - notifications. The scheduler communicates with workers only through - the standard ``build_connector_meta`` / ``update_connector_output`` - hooks. - """ - - _TRANSFER_MODE: str = "push" - - def __init__( - self, - vllm_config: VllmConfig, - engine_id: str, - kv_cache_config: KVCacheConfig, - ): - super().__init__(vllm_config, engine_id, kv_cache_config) - if self.is_bidirectional_kv_xfer_enabled: - raise NotImplementedError( - "Bidirectional KV transfer is not supported for NIXL push connector." - ) - - # D-side: registration data to pass to D workers via metadata on - # the next ``build_connector_meta`` call. - self._push_pending_registrations: dict[ReqId, dict[str, Any]] = {} - - # D-side: track the wall-clock deadline for each registered request - # to detect "registered but never fulfilled" failures (e.g. the P - # node disappeared after registration). Keyed by D request_id. - self._push_registration_deadlines: dict[ReqId, float] = {} - - # P-side: block IDs for finished requests, kept for the lease and - # used to drive ``has_pending_push_work``. - self._finished_request_blocks: dict[ReqId, BlockIds] = {} - # P-side: newly finished blocks to ship to P workers on next step. - self._newly_finished_push_blocks: dict[ReqId, BlockIds] = {} - - # Soft watchdog timeout (seconds) for D-side registrations that - # never receive a push completion. Defaults to the existing - # decoder KV blocks TTL so behaviour matches the lease. - assert vllm_config.kv_transfer_config is not None - self._push_registration_timeout: float = float( - vllm_config.kv_transfer_config.get_from_extra_config( - "push_registration_timeout", - self.decoder_kv_blocks_ttl, - ) - ) - - def get_num_new_matched_tokens( - self, request: Request, num_computed_tokens: int - ) -> tuple[int, bool]: - """In push mode, D doesn't pull — it registers blocks and waits. - - However, we still need to handle the do_remote_prefill case where D - needs to know how many tokens will be pushed. - """ - params = request.kv_transfer_params - logger.debug( - "NixlPushConnector get_num_new_matched_tokens: " - "num_computed_tokens=%s, kv_transfer_params=%s", - num_computed_tokens, - params, - ) - - if params is not None and params.get("do_remote_prefill"): - token_ids = request.prompt_token_ids or [] - actual = self._get_remote_prefill_token_count(len(token_ids)) - count = actual - num_computed_tokens - if count > 0: - return count, True - - return 0, False - - def update_state_after_alloc( - self, request: Request, blocks: KVCacheBlocks, num_external_tokens: int - ): - """In push mode, D stores registration data for the worker to send - to P via NIXL notification (deferred to ``build_connector_meta``). - """ - params = request.kv_transfer_params - logger.debug( - "NixlPushConnector update_state_after_alloc: " - "num_external_tokens=%s, kv_transfer_params=%s", - num_external_tokens, - params, - ) - - if not params: - return - - # P side: track the request as in-batch so the lease accounting - # matches what the worker expects on the next step. - if params.get("do_remote_decode"): - self._reqs_in_batch.add(request.request_id) - - # P side with host-buffer offload: defer save to the worker. - if self.use_host_buffer and params.get("do_remote_decode"): - self._reqs_need_save[request.request_id] = request - return - - # D side: only act on the first call (``do_remote_prefill`` is - # unset on re-entry by the marker below). - if not params.get("do_remote_prefill"): - return - - if num_external_tokens <= 0: - # Nothing to receive: full prefix-cache hit on D, no - # registration to stage. - return - - # First-pass D path: stash registration data the worker will - # ship to P on the next ``build_connector_meta`` cycle. - logger.debug( - "KV PUSH mode: D node storing registration for request %s", - request.request_id, - ) - local_block_ids: BlockIds = blocks.get_unhashed_block_ids_all_groups() - local_block_ids = self.get_exchange_clipped_blocks(local_block_ids) - - # ``remote_*`` fields are P's coordinates (from D's perspective). - # ``decode_*`` fields are D's own info that P needs for the - # reverse handshake before WRITE-ing. - self._push_pending_registrations[request.request_id] = { - "request_id": request.request_id, - "decode_engine_id": self.engine_id, - "decode_host": self.side_channel_host, - "decode_port": self.side_channel_port, - "decode_tp_size": (self.vllm_config.parallel_config.tensor_parallel_size), - "local_block_ids": local_block_ids, - "remote_engine_id": params["remote_engine_id"], - "remote_host": params["remote_host"], - "remote_port": params["remote_port"], - "remote_tp_size": params["tp_size"], - "remote_pp_size": params.get("pp_size", 1), - } - self._push_registration_deadlines[request.request_id] = ( - time.perf_counter() + self._push_registration_timeout - ) - # In push mode D doesn't know P's blocks; P determines them - # from the registration. We still track the request as - # needing recv so the engine waits for P's WRITE completion. - # ``remote_block_ids`` is also seeded to an empty tuple so the - # base scheduler's ``add_new_req_to_recv`` can build the - # ReqMeta without a KeyError — the actual remote block IDs are - # learned by P over the NIXL handshake at WRITE time. - params["remote_block_ids"] = () - self._reqs_need_recv[request.request_id] = (request, local_block_ids, (), False) - - # Mark as processed so a re-entry (e.g. preemption + reschedule) - # doesn't re-stage the registration. - params["do_remote_prefill"] = False - - def request_finished( - self, - request: Request, - block_ids: BlockIds, - ) -> tuple[bool, dict[str, Any] | None]: - """Push-mode request_finished: stores blocks for workers.""" - from vllm.v1.request import RequestStatus - - params = request.kv_transfer_params - logger.debug( - "NixlPushConnector request_finished(%s), request_status=%s, " - "kv_transfer_params=%s", - request.request_id, - request.status, - params, - ) - if not params: - return False, None - - is_p_node = bool(params.get("do_remote_decode")) - - self._stop_heartbeat(request.request_id) - # Drop any pending registration deadline; the request either - # completed or was cancelled. - self._push_registration_deadlines.pop(request.request_id, None) - - if params.get("do_remote_prefill"): - # ``do_remote_prefill`` is still set, which means - # ``update_state_after_alloc`` never ran (it would have - # flipped this flag to False). The request was aborted - # before it could be scheduled — e.g. rejected at the D - # serving layer via abort_immediately. To keep P from - # stranding the prefill blocks, we still register an empty - # recv so the worker emits a notif that lets P free them. - # Seed remote_block_ids so add_new_req_to_recv won't KeyError. - params["remote_block_ids"] = () - self._reqs_need_recv[request.request_id] = (request, [], (), False) - params["do_remote_prefill"] = False - return False, None - - # Push connector only acts on the P-side terminal path; D-side - # finishing without a remote prefill is a no-op. - if not is_p_node: - return False, None - - if request.status not in ( - RequestStatus.FINISHED_LENGTH_CAPPED, - RequestStatus.FINISHED_STOPPED, - ): - self._reqs_not_processed.add(request.request_id) - self._reqs_need_save.pop(request.request_id, None) - return False, None - - delay_free_blocks = any(len(group) > 0 for group in block_ids) - remote_num_tokens = 0 - if delay_free_blocks: - logger.debug( - "NixlPushConnector request_finished(%s) waiting for %d seconds " - "before releasing blocks", - request.request_id, - self._kv_lease_duration, - ) - self._reqs_need_send[request.request_id] = ( - time.perf_counter() + self._kv_lease_duration - ) - - block_ids = self.get_exchange_clipped_blocks(block_ids) - remote_num_tokens = request.num_computed_tokens - - # Store finished blocks for worker-level matching with D - # registrations (via NIXL notifications). - self._finished_request_blocks[request.request_id] = block_ids - self._newly_finished_push_blocks[request.request_id] = block_ids - - return delay_free_blocks, dict( - do_remote_prefill=True, - do_remote_decode=False, - remote_block_ids=block_ids, - remote_engine_id=self.engine_id, - remote_request_id=request.request_id, - remote_host=self.side_channel_host, - remote_port=self.side_channel_port, - tp_size=self.transfer_tp_size, - dcp_size=self.vllm_config.parallel_config.decode_context_parallel_size, - pp_size=self.vllm_config.parallel_config.pipeline_parallel_size, - remote_num_tokens=remote_num_tokens, - transfer_mode=self._TRANSFER_MODE, - ) - - def build_connector_meta( - self, - scheduler_output: SchedulerOutput, - ) -> KVConnectorMetadata: - meta = super().build_connector_meta(scheduler_output) - assert isinstance(meta, NixlConnectorMetadata) - - # Watchdog: any D-side registration whose deadline has passed without - # a corresponding push completion is treated as failed and cleaned up. - # The corresponding request is already tracked via _reqs_need_recv; - # the engine layer will eventually time it out via the lease, but we - # at least drop the stale registration so we don't keep retrying. - now = time.perf_counter() - # Deadlines are inserted in non-decreasing order (monotonic clock + - # constant timeout, armed once per request), and dict insertion order - # is preserved across key deletions, so we can stop at the first - # not-yet-expired entry instead of scanning the whole dict. - expired = [] - for rid, deadline in self._push_registration_deadlines.items(): - if deadline > now: - break - expired.append(rid) - for rid in expired: - self._push_registration_deadlines.pop(rid, None) - # Avoid resending a registration that already timed out. - self._push_pending_registrations.pop(rid, None) - logger.warning( - "NixlPushConnector: registration for request %s timed out " - "after %.1fs without a push completion", - rid, - self._push_registration_timeout, - ) - - # D side: package pending registrations for D workers to send out. - if self._push_pending_registrations: - meta.push_registrations = dict(self._push_pending_registrations) - self._push_pending_registrations.clear() - - # P side: package newly finished blocks for P workers to match against - # any D registrations they have received via NIXL notifications. - if self._newly_finished_push_blocks: - meta.push_finished_blocks = dict(self._newly_finished_push_blocks) - self._newly_finished_push_blocks.clear() - - return meta - - def has_pending_push_work(self) -> bool: - # Keep the engine main loop alive while we have: - # - finished P blocks awaiting WRITE completion, or - # - pending D registrations the worker has not yet shipped, or - # - newly finished blocks not yet shipped to P workers. - return bool(self._finished_request_blocks or self._push_pending_registrations) - - def update_connector_output(self, connector_output: KVConnectorOutput) -> None: - """Clean up finished request blocks after push completes.""" - super().update_connector_output(connector_output) - for req_id in connector_output.finished_sending or (): - self._finished_request_blocks.pop(req_id, None) - # On D side, finished_recving means the push completed; clear the - # watchdog so we don't trip an expiration on a fulfilled request. - for req_id in connector_output.finished_recving or (): - self._push_registration_deadlines.pop(req_id, None) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/push_worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/push_worker.py deleted file mode 100644 index 38e7d0fbb7..0000000000 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/push_worker.py +++ /dev/null @@ -1,827 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Push-specific (WRITE) worker-side logic for the NIXL connector. - -A dedicated ``nixl-push-writer`` thread owns all push-related NIXL ops: -calls ``get_new_notifs`` (routing PUSH_REG internally; HB / completion -notifs are forwarded to the engine main thread), sends PUSH_REG via -``send_notif``, matches D registrations with P finished blocks, and -issues WRITE transfers via ``make_prepped_xfer`` / ``transfer``. - -The engine main thread feeds the writer through queues: -``_reg_send_inbox`` (D-side regs to send), ``_finished_blocks_inbox`` -(P-side blocks from metadata) and ``_pending_completion_notifs`` -(non-PUSH_REG notifs forwarded back for HB / completion accounting). -The handshake-completion callback feeds ``_deferred_push_inbox`` with -matched pushes whose P→D handshake has finished so the writer can -(re-)issue the WRITE without ever blocking on the network. - -Wake model: the writer self-polls every -``_PUSH_WRITER_POLL_INTERVAL_MS`` only while it has unmatched -``_push_finished_blocks`` (i.e. P-side blocks waiting for a D PUSH_REG -notif that has no other wake source). All other progress is -event-driven: the engine main thread sets ``_push_writer_wake`` from -``start_load_kv`` (when handing it new work) and from ``get_finished`` -(so each engine step gives the writer a chance to drain NIXL notifs); -the handshake-completion callback sets the same event after a deferred -PUSH_REG send or a deferred push WRITE has been queued. When a request's -lease expires (the base worker reports it via ``done_sending``) or the WRITE completes, -``get_finished`` enqueues an eviction onto ``_evict_finished_inbox`` so -the writer drops any leftover ``_push_finished_blocks`` / -``_pending_d_registrations`` and stops self-polling. -""" - -import queue -import threading -import time -from collections import defaultdict -from concurrent.futures import Future -from typing import TYPE_CHECKING, Any - -import msgspec - -from vllm.distributed.kv_transfer.kv_connector.utils import BlockIds -from vllm.distributed.kv_transfer.kv_connector.v1.base import ( - KVConnectorTransferResults, -) -from vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker import ( - NixlBaseConnectorWorker, -) -from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import ( - PUSH_REG_NOTIF_PREFIX, - NixlConnectorMetadata, - RemoteMeta, - ReqId, - ReqMeta, - TransferHandle, -) -from vllm.distributed.kv_transfer.kv_connector.v1.nixl.tp_mapping import ( - ReadSpec, - _is_attention_spec, -) -from vllm.distributed.kv_transfer.kv_connector.v1.nixl.utils import get_base_request_id -from vllm.logger import init_logger - -if TYPE_CHECKING: - import torch - - from vllm.config import VllmConfig - from vllm.v1.kv_cache_interface import KVCacheConfig - -logger = init_logger(__name__) - -# Writer-thread poll cadence while there is in-flight push state. When -# fully idle, the writer blocks on a wake event signalled by the engine -# main thread (start_load_kv / get_finished). Smaller -> lower latency -# while active, slightly more CPU. -_PUSH_WRITER_POLL_INTERVAL_MS = 1.0 - - -class NixlPushConnectorWorker(NixlBaseConnectorWorker): - """Push-specific (WRITE) worker logic. See module docstring.""" - - # Distinguishes push from pull in the NIXL compatibility hash. - _TRANSFER_MODE: str = "push" - - _supports_pp_hma = True - - def __init__( - self, - vllm_config: "VllmConfig", - engine_id: str, - kv_cache_config: "KVCacheConfig", - ): - super().__init__(vllm_config, engine_id, kv_cache_config) - - # Heartbeat handshakes to a PP-sharded producer must be notif-only, - # like the PUSH_REG path. - self._hb_handshake_notif_only = True - - # Push-specific state. - # P-side: outgoing WRITE handles awaiting completion, keyed by - # request_id. Mutated by writer (submit) and main thread - # (``_pop_done_transfers``); guarded by - # ``_sending_transfers_lock``. - self._sending_transfers = defaultdict[ReqId, list[TransferHandle]](list) - self._send_failures: set[ReqId] = set() - self._sending_transfers_lock = threading.Lock() - - # Writer-thread owned matching state. - # P-side: finished request blocks received from scheduler metadata - # that have not yet been matched with an incoming D registration. - self._push_finished_blocks: dict[ReqId, BlockIds] = {} - # P-side: D registrations received via NIXL notification that have - # not yet been matched with a finished P request. - self._pending_d_registrations: dict[ReqId, dict[str, Any]] = {} - - # Cross-thread channels. - self._reg_send_inbox: queue.Queue[tuple[str, dict[str, Any]]] = queue.Queue() - self._finished_blocks_inbox: queue.Queue[tuple[str, BlockIds]] = queue.Queue() - self._pending_completion_notifs: queue.Queue[bytes] = queue.Queue() - # Main thread → writer: req_ids whose lease has expired or whose - # WRITE has completed. Writer drops them from - # ``_push_finished_blocks`` so an unmatched entry doesn't keep the - # writer busy-polling forever. - self._evict_finished_inbox: queue.Queue[str] = queue.Queue() - # Handshakes that have just completed and are ready for the WRITE on wthread - self._deferred_push_inbox = queue.Queue[tuple[str, BlockIds, dict[str, Any]]]() - - # Wake signal from engine main thread (start_load_kv / get_finished). - # Writer self-polls at _PUSH_WRITER_POLL_INTERVAL_MS while it has - # active in-flight state; otherwise it blocks until signalled. - self._push_writer_wake = threading.Event() - - self._push_writer_stop = threading.Event() - self._push_writer_thread: threading.Thread | None = None - - # --- Lifecycle ----------------------------------------------------- # - - def register_kv_caches(self, kv_caches: dict[str, "torch.Tensor"]): - super().register_kv_caches(kv_caches) - if self._mixed_mem_types: - raise NotImplementedError( - "NixlPushConnector does not support mixed-memory KV caches" - ) - if self._push_writer_thread is None: - self._push_writer_thread = threading.Thread( - target=self._push_writer_loop, - daemon=True, - name="nixl-push-writer", - ) - self._push_writer_thread.start() - logger.info("nixl-push-writer thread started (rank=%d)", self.tp_rank) - - def shutdown(self) -> None: - self._push_writer_stop.set() - # Unblock the writer if it's waiting in the no-active-state branch. - self._push_writer_wake.set() - if self._push_writer_thread is not None: - self._push_writer_thread.join(timeout=2) - self._push_writer_thread = None - with self._sending_transfers_lock: - for handles in self._sending_transfers.values(): - for handle in handles: - self.nixl_wrapper.release_xfer_handle(handle) - self._sending_transfers.clear() - self._send_failures.clear() - super().shutdown() - - # --- Engine-main-thread entry point -------------------------------- # - - def start_load_kv(self, metadata: NixlConnectorMetadata): - """Pre-process metadata; defer NIXL ops to the writer thread.""" - if self.pcp_rank > 0 and not self.pcp_dcp_sharded: - return - - # D-side: track reqs waiting for P to push. - for req_id, meta in metadata.reqs_to_recv.items(): - meta.local_physical_block_ids = self._logical_to_kernel_block_ids( - meta.local_block_ids, self._physical_blocks_per_logical_kv_block - ) - assert meta.remote is not None - remote_engine_id = meta.remote.engine_id - # Update last activity from this remote (same as pull), but only for - # an already-connected engine so we never leave an _engine_last_active - # entry without a _remote_agents entry (breaks _cleanup_remote_engine). - if remote_engine_id in self._remote_agents: - self._engine_last_active[remote_engine_id] = time.perf_counter() - - logger.debug( - "start_load_kv (push) for request %s from remote engine %s. " - "Num local_block_ids: %s. Num remote_block_ids: %s. ", - req_id, - remote_engine_id, - len(meta.local_physical_block_ids), - len(meta.remote.block_ids), - ) - self._recving_metadata[req_id] = meta - - # --- D-side: registrations to send to P via NIXL --- - if metadata.push_registrations: - for req_id, reg_data in metadata.push_registrations.items(): - self._reg_send_inbox.put((req_id, reg_data)) - self._push_writer_wake.set() - - # --- P-side: newly finished blocks awaiting a D registration match --- - if metadata.push_finished_blocks: - for req_id, block_ids in metadata.push_finished_blocks.items(): - self._finished_blocks_inbox.put((req_id, block_ids)) - self._push_writer_wake.set() - - # Batch + lease tracking (same as pull). - for req_id in metadata.reqs_in_batch: - self._reqs_to_process.add(req_id) - for req_id in metadata.reqs_not_processed: - self._reqs_to_process.discard(req_id) - assert req_id not in self._reqs_to_send - # Rebase scheduler-clock deadlines onto this worker's clock — see the - # equivalent block in pull_worker.start_load_kv for the rationale. - now_local = time.perf_counter() - for req_id, expiration_time in metadata.reqs_to_send.items(): - if req_id in self._reqs_to_process: - if metadata.scheduler_clock: - expiration_time = now_local + ( - expiration_time - metadata.scheduler_clock - ) - self._reqs_to_send[req_id] = expiration_time - - # Heartbeats still leave from the main thread (base worker behaviour). - self._send_heartbeats(metadata) - - # --- Writer thread ------------------------------------------------- # - - def _push_writer_loop(self) -> None: - sleep_s = _PUSH_WRITER_POLL_INTERVAL_MS / 1000.0 - - while not self._push_writer_stop.is_set(): - try: - # 1. D registrations to send. - while True: - try: - rid, rd = self._reg_send_inbox.get_nowait() - except queue.Empty: - break - self._send_registration_to_p(rid, rd) - - # 2. Deferred P→D pushes whose handshake just completed; do xfer now - while True: - try: - rid, blocks, rd = self._deferred_push_inbox.get_nowait() - except queue.Empty: - break - self._do_start_push_kv(rid, blocks, rd) - - # 3. P-side finished blocks; match against pending regs. - while True: - try: - rid, blocks = self._finished_blocks_inbox.get_nowait() - except queue.Empty: - break - matched = self._pop_matching_registration(rid) - if matched is not None: - self._do_start_push_kv(rid, blocks, matched) - else: - self._push_finished_blocks[rid] = blocks - - # 3b. Evict finished blocks for requests that have either - # completed (WRITE acknowledged) or whose lease expired - # without a D registration. Drop pending registrations - # for the same reason so we don't leak state. - while True: - try: - rid = self._evict_finished_inbox.get_nowait() - except queue.Empty: - break - self._push_finished_blocks.pop(rid, None) - self._pending_d_registrations.pop(rid, None) - - # 4. NIXL notifs: route PUSH_REG; forward the rest. - for notifs in self.nixl_wrapper.get_new_notifs().values(): - for notif in notifs: - if notif.startswith(PUSH_REG_NOTIF_PREFIX): - self._handle_push_reg_notif(notif) - else: - self._pending_completion_notifs.put(notif) - except Exception: - logger.exception("nixl-push-writer error; continuing") - - # Self-poll only while there is no other wake source: P-side - # finished blocks waiting for a D PUSH_REG match. All other - # progress is event-driven (see module docstring). - if self._push_finished_blocks: - self._push_writer_stop.wait(timeout=sleep_s) - else: - self._push_writer_wake.wait() - self._push_writer_wake.clear() - - def _handle_push_reg_notif(self, notif: bytes) -> None: - try: - reg_data = msgspec.msgpack.decode(notif[len(PUSH_REG_NOTIF_PREFIX) :]) - except Exception: - logger.exception("Failed to decode PUSH_REG notification payload") - return - rid = reg_data.get("request_id") if isinstance(reg_data, dict) else None - if not isinstance(rid, str): - logger.warning("PUSH_REG notif missing request_id; dropping") - return - - match = self._pop_matching_finished_blocks(rid) - if match is not None: - fin_id, blocks = match - self._do_start_push_kv(fin_id, blocks, reg_data) - else: - self._pending_d_registrations[rid] = reg_data - - # --- D-side registration send (writer thread) ---------------------- # - - def _send_registration_to_p( - self, - req_id: str, - reg_data: dict[str, Any], - ) -> None: - """Handshake (if needed) then send PUSH_REG. ``send_notif`` always - executes on the writer; the handshake runs on the background executor - and the request is re-queued onto ``_reg_send_inbox`` once it - completes (at which point ``_ensure_handshake`` returns ``None`` and we - send directly).""" - remote_pp_size = reg_data.get("remote_pp_size", 1) - fut = self._ensure_handshake( - reg_data["remote_engine_id"], - reg_data["remote_host"], - reg_data["remote_port"], - reg_data["remote_tp_size"], - pp_size=remote_pp_size, - # D only ever sends PUSH_REG notifs to P and never reads or writes - # P's memory in push mode, so it never needs the transfer - # descriptors set up by the full add_remote_agent path. - notif_agents_only=True, - ) - if fut is None: - self._do_send_reg_notif(req_id, reg_data) - return - - def _on_handshake( - f: Future[tuple[dict[tuple[int, int], str], float]], - rid: str = req_id, - rd: dict[str, Any] = reg_data, - ) -> None: - try: - f.result() - except Exception as e: - self._log_failure( - failure_type="push_reg_handshake_failed", req_id=rid, error=e - ) - self._failed_recv_reqs.put(rid) - return - # Re-queue for the writer to send now that the handshake is done. - self._reg_send_inbox.put((rid, rd)) - # Wake the writer so it sends the PUSH_REG promptly even if - # otherwise parked. - self._push_writer_wake.set() - - fut.add_done_callback(_on_handshake) - - def _do_send_reg_notif(self, req_id: str, reg_data: dict[str, Any]) -> None: - engine_id = reg_data["remote_engine_id"] - notif_msg = PUSH_REG_NOTIF_PREFIX + msgspec.msgpack.encode(reg_data) - # _remote_agents is mutated on other threads; snapshot under the lock. - with self._handshake_lock: - agents = dict(self._remote_agents.get(engine_id) or {}) - if not agents: - logger.error( - "No remote agents for engine %s; cannot send registration for %s", - engine_id, - req_id, - ) - self.xfer_stats.record_failed_notification() - self._failed_recv_reqs.put(req_id) - return - for rank, agent_name in agents.items(): - try: - self.nixl_wrapper.send_notif(agent_name, notif_msg=notif_msg) - except Exception as e: - self._log_failure( - failure_type="push_reg_notif_failed", - req_id=req_id, - error=e, - remote_rank=rank, - ) - self.xfer_stats.record_failed_notification() - # Earlier registrations may still trigger WRITEs into D's blocks. - # Keep the receive pending until those writes are finished. - logger.debug( - "Sent PUSH_REG for %s to engine %s (%dB)", req_id, engine_id, len(notif_msg) - ) - - # --- Matching helpers --------------------------------------------- # - - def _pop_matching_registration(self, request_id: str) -> dict[str, Any] | None: - """Pop the D-side registration matching *request_id*. - - Exact key first, then a match after stripping the random suffix from - both sides. No match leaves the request unmatched (push not started). - """ - data = self._pending_d_registrations.pop(request_id, None) - if data is not None: - return data - base_id = get_base_request_id(request_id) - for reg_id in list(self._pending_d_registrations): - if get_base_request_id(reg_id) == base_id: - return self._pending_d_registrations.pop(reg_id) - return None - - def _pop_matching_finished_blocks( - self, request_id: str - ) -> tuple[str, BlockIds] | None: - """Pop the P-side finished blocks matching *request_id*. - - Same lookup as ``_pop_matching_registration``: exact key, then a - match after stripping the random suffix from both sides. - """ - blocks = self._push_finished_blocks.pop(request_id, None) - if blocks is not None: - return request_id, blocks - base_id = get_base_request_id(request_id) - for fin_id in list(self._push_finished_blocks): - if get_base_request_id(fin_id) == base_id: - return fin_id, self._push_finished_blocks.pop(fin_id) - return None - - # --- WRITE transfer logic (writer thread) ------------------------- # - - def _do_start_push_kv( - self, - request_id: str, - local_block_ids: BlockIds, - registration_data: dict[str, Any], - ) -> None: - """Start push-based KV transfer from P worker to D node. - - The P→D handshake runs on the base worker's background executor. - If it isn't ready yet we register a completion callback, defer the - WRITE, and re-drive this request via ``_deferred_push_inbox`` once - the handshake resolves -- so the writer thread never blocks on the - network (mirrors ``_send_registration_to_p``). - """ - if not local_block_ids: - logger.warning("No local blocks to push for request %s", request_id) - return - - # ``local_block_ids`` are P's logical block IDs; ``remote_block_ids`` - # (D's, from the PUSH_REG notif) are also logical. - decode_engine_id = registration_data["decode_engine_id"] - remote_block_ids = registration_data["local_block_ids"] - decode_request_id = registration_data["request_id"] - - # Runs on the background executor; defer the WRITE until it's ready. - fut = self._ensure_handshake( - decode_engine_id, - registration_data["decode_host"], - registration_data["decode_port"], - registration_data["decode_tp_size"], - ) - if fut is not None: - - def _on_handshake( - f: Future[tuple[dict[tuple[int, int], str], float]], - rid: str = request_id, - blocks: BlockIds = local_block_ids, - rd: dict[str, Any] = registration_data, - ) -> None: - if (e := f.exception()) is not None: - # The engine reclaims the blocks via the TTL so we dont free here - self._log_failure( - failure_type="push_handshake_failed", req_id=rid, error=e - ) - return - self._deferred_push_inbox.put((rid, blocks, rd)) - self._push_writer_wake.set() - - fut.add_done_callback(_on_handshake) - return - # Keep the engine alive while it is actively receiving pushes, mirroring - # how pull-mode transfers touch _engine_last_active in start_load_kv. - self._engine_last_active[decode_engine_id] = time.perf_counter() - - logical_local = self._as_grouped_block_ids(local_block_ids) - logical_remote = self._as_grouped_block_ids(remote_block_ids) - physical_local = self._logical_to_kernel_block_ids( - logical_local, self._physical_blocks_per_logical_kv_block - ) - - push_meta = ReqMeta( - local_block_ids=logical_local, - local_physical_block_ids=physical_local, - tp_size=self.world_size, - remote=RemoteMeta( - block_ids=logical_remote, - host="", - port=0, - engine_id=decode_engine_id, - request_id=decode_request_id, - ), - ) - - t0 = time.perf_counter() - self._xfer_blocks_for_req(req_id=request_id, meta=push_meta) - elapsed_ms = (time.perf_counter() - t0) * 1000.0 - if elapsed_ms > 200.0: - logger.warning( - "_do_start_push_kv for %s took %.1fms (slow NIXL submission)", - request_id, - elapsed_ms, - ) - - @staticmethod - def _as_grouped_block_ids(block_ids: BlockIds) -> BlockIds: - """Normalise a sequence of block IDs to a tuple-of-groups shape. - - ``BlockIds`` is canonically a tuple of per-group lists, but some - registration payloads collapse a single-group case to a flat - list. Re-wrap that case so downstream group-aware helpers see a - consistent shape.""" - if block_ids and not isinstance(block_ids[0], (list, tuple)): - return (list(block_ids),) - return block_ids - - def _xfer_blocks_for_req(self, req_id: str, meta: ReqMeta): - """Issue WRITE transfers to one or more remote TP ranks.""" - assert meta.remote is not None and self.transfer_topo is not None - engine_id = meta.remote.engine_id - plan = self.tp_mappings[engine_id] - remote_info = self.transfer_topo.get_engine_info(engine_id) - tp_ratio = self.transfer_topo.tp_ratio(remote_info.remote_tp_size) - - # Expand D's logical IDs using the ratio learned during the - # NIXL handshake. ``meta`` is freshly built by - # ``_do_start_push_kv`` so mutating it here is safe. - meta.remote.block_ids = self._logical_to_kernel_block_ids( - meta.remote.block_ids, - remote_info.remote_physical_blocks_per_logical, - ) - remote_block_ids = meta.remote.block_ids - local_block_ids = meta.local_physical_block_ids - local_region_groups = self.region_group_ids - remote_region_groups = self.dst_region_group_ids[engine_id] - groups_differ = local_region_groups != remote_region_groups - if groups_differ and not self._transfer_layer_group_ids: - raise NotImplementedError( - "NixlPushConnector does not support different producer and " - "consumer cache-group layouts" - ) - - # MLA latent is replicated across D's TP ranks: the tp-mapping - # collapses it to one rank (fine for reads), but push must WRITE every - # D rank or the rest decode stale KV. For hybrid MLA+SSM the sharded - # SSM state already targets every covered D rank, so only the - # attention groups need widening; pure MLA writes to all handshaked - # ranks (only the dst differs per rank). - replicate_attn = self.use_mla and tp_ratio < 0 - if replicate_attn and not self._has_mamba: - assert len(plan.all_source_ranks) == 1 - write_ranks = sorted(self.dst_xfer_side_handles[engine_id]) - else: - write_ranks = list(plan.all_source_ranks) - - num_groups = len(local_block_ids) - - def group_ids(block_ids: BlockIds, rank: int) -> BlockIds: - return [ - list(block_ids[g]) - if (self._is_csa_linear and tp_ratio < 0) - or (replicate_attn and _is_attention_spec(self._group_spec_types[g])) - or rank in plan.source_ranks_per_group[g] - else [] - for g in range(num_groups) - ] - - read_specs = [ - ReadSpec( - remote_rank=rank, - local_block_ids=group_ids(local_block_ids, rank), - remote_block_ids=group_ids(remote_block_ids, rank), - ) - for rank in write_ranks - ] - - handles: list[int] = [] - for i, spec in enumerate(read_specs): - remote_block_size = remote_info.remote_block_size - logger.debug( - "Remote agent %s available, calling _xfer_blocks" - " on remote rank %s with remote block size %s for req %s", - meta.remote.engine_id, - spec.remote_rank, - remote_block_size, - req_id, - ) - if tp_ratio < 0 and (not self.use_mla or len(plan.all_source_ranks) > 1): - # Multiple targets: write each rank its chunk of local memory. - # Hybrid MLA+SSM also lands here: its split handles replicate - # the attention descriptors and chunk only the SSM state. - split_key = (tp_ratio, remote_block_size) - local_xfer_side_handle = self.src_xfer_handles_by_tp_ratio[split_key][i] - else: - local_xfer_side_handle = self.src_xfer_handles_by_block_size[ - remote_block_size - ] - - remote_xfer_side_handle = self.dst_xfer_side_handles[meta.remote.engine_id][ - spec.remote_rank - ] - - handle = self._xfer_blocks( - read_spec=spec, - request_id=req_id, - dst_engine_id=meta.remote.engine_id, - remote_request_id=meta.remote.request_id, - local_xfer_side_handle=local_xfer_side_handle, - remote_xfer_side_handle=remote_xfer_side_handle, - ) - if handle is not None: - handles.append(handle) - - # Publish all the request's WRITE handles in one locked update: a - # partial set would let ``_pop_done_transfers`` finish the request - # early, then double-report it as the remaining writes land. - if handles: - with self._sending_transfers_lock: - self._sending_transfers[req_id].extend(handles) - - def _xfer_blocks( - self, - read_spec: ReadSpec, - dst_engine_id: str, - request_id: str, - remote_request_id: str, - local_xfer_side_handle: int, - remote_xfer_side_handle: int, - ) -> int | None: - """Post a WRITE point-to-point xfer request. - - Returns the in-flight transfer handle (so the caller can track all of - a request's handles atomically), or ``None`` if nothing was submitted. - """ - assert self.transfer_topo is not None - remote_rank = read_spec.remote_rank - local_block_ids = read_spec.local_block_ids - remote_block_ids = read_spec.remote_block_ids - - remote_info = self.transfer_topo.get_engine_info(dst_engine_id) - block_size_ratio = self.transfer_topo.block_size_ratio( - remote_info.remote_block_size - ) - if block_size_ratio > 1: - local_block_ids, remote_block_ids = ( - self._map_block_ids_for_block_size_ratio( - local_block_ids, remote_block_ids, block_size_ratio - ) - ) - - notif_id = f"{remote_request_id}:{self.world_size}".encode() - - if len(local_block_ids) == 0: - logger.warning("No blocks to push for request %s", request_id) - return None - - # Prefix caching: D allocated only uncached blocks, so on a partial hit it - # sends fewer than P's. End-trim P's blocks to that same suffix so we WRITE only - # the uncomputed tail into D's slots. Runs on kernel ids, post-expansion. - remote_block_ids, local_block_ids = self._apply_prefix_caching( - decode_block_ids=remote_block_ids, - prefill_block_ids=local_block_ids, - decode_physical_per_logical=remote_info.remote_physical_blocks_per_logical, - prefill_physical_per_logical=self._physical_blocks_per_logical_kv_block, - ) - - local_block_ids = list(local_block_ids) - remote_block_ids = list(remote_block_ids) - assert len(local_block_ids) == len(remote_block_ids), ( - f"push group-count mismatch for {request_id}: {len(local_block_ids)} " - f"local vs {len(remote_block_ids)} remote groups" - ) - for i in range(len(local_block_ids)): - assert len(local_block_ids[i]) == len(remote_block_ids[i]), ( - f"push block-count mismatch for {request_id} group {i}: " - f"{len(local_block_ids[i])} local vs " - f"{len(remote_block_ids[i])} remote blocks" - ) - - # Get descs ids. - remote_block_descs_ids = self._compute_desc_ids( - block_ids=remote_block_ids, - dst_num_blocks=self.dst_num_blocks[dst_engine_id], - block_size_ratio=None, - physical_blocks_per_logical=remote_info.remote_physical_blocks_per_logical, - region_num_blocks=self.dst_region_num_blocks[dst_engine_id], - region_group_ids=self.dst_region_group_ids[dst_engine_id], - uses_region_group_mapping=self.dst_uses_region_group_mapping[dst_engine_id], - ) - local_block_descs_ids = self._compute_desc_ids( - block_ids=local_block_ids, - dst_num_blocks=self.dst_num_blocks[self.engine_id], - block_size_ratio=block_size_ratio, - physical_blocks_per_logical=self._physical_blocks_per_logical_kv_block, - region_num_blocks=self.dst_region_num_blocks[self.engine_id], - region_group_ids=self.region_group_ids, - uses_region_group_mapping=self._uses_region_group_mapping, - ) - - assert len(local_block_descs_ids) == len(remote_block_descs_ids) - - handle = None - try: - handle = self.nixl_wrapper.make_prepped_xfer( - "WRITE", - local_xfer_side_handle, - local_block_descs_ids, - remote_xfer_side_handle, - remote_block_descs_ids, - notif_msg=notif_id, - ) - self.nixl_wrapper.transfer(handle) - # Caller tracks the handle (atomically with the request's other - # writes) so P can free blocks once all of them are done. - return handle - except Exception as e: - self._log_failure( - failure_type="transfer_setup_failed", - req_id=request_id, - msg="Push WRITE submission failed; releasing handle", - error=e, - dst_engine_id=dst_engine_id, - remote_rank=remote_rank, - ) - # On the P side this WRITE failure is purely outbound; we - # don't have a ``_recving_metadata`` entry to invalidate, so - # we just release the handle and let the engine reschedule - # via the lease / watchdog. - with self._sending_transfers_lock: - self._send_failures.add(request_id) - if not self._handle_failed_transfer(request_id, handle): - return handle - return None - - # --- Notification handling on engine main thread ------------------ # - - def _get_new_notifs(self) -> set[str]: - """Drain HB / completion notifs forwarded by the writer thread. - - The writer owns ``nixl_wrapper.get_new_notifs`` for push; PUSH_REG - notifs are handled there. P learns of finished pushes from its WRITE - handles, so this never reports finished sends. - """ - while True: - try: - notif = self._pending_completion_notifs.get_nowait() - except queue.Empty: - break - - msg = notif.decode("utf-8") - if msg.startswith("HB:"): - self._handle_heartbeat(msg[3:]) - continue - - req_id, tp_size = msg.rsplit(":", 1) - - if (meta := self._recving_metadata.get(req_id)) is None: - # The lease may have expired before the notif arrived. - self.xfer_stats.record_notification_after_expiry() - logger.error( - "Unrecognized request %s notif (may have expired).", req_id - ) - continue - # Consumer waits for one notif per producer rank writing - # here: pp_size stages * producers-per-consumer (>1 when - # producer TP > consumer TP; tp_size is the producer TP). - producers_per_consumer = max(1, int(tp_size) // self.world_size) - expected_notifs = meta.pp_size * producers_per_consumer - self.consumer_notification_counts_by_req[req_id] += 1 - if self.consumer_notification_counts_by_req[req_id] < expected_notifs: - continue - del self.consumer_notification_counts_by_req[req_id] - # P drove the transfer (we own no NIXL handle), so - # materialise an empty ``_recving_transfers`` entry for - # ``_pop_done_transfers`` to report done. - self._recving_transfers.setdefault(req_id, []) - return set() - - def get_transfer_results(self) -> KVConnectorTransferResults: - # Engine main thread asking for completions: also wake the writer - # so it gets a chance to drain NIXL notifs (heartbeats, completion - # notifs, late PUSH_REGs) even if it had been parked. - self._push_writer_wake.set() - - results = super().get_transfer_results() - done_sending = results.finished_sending - - # ``_pop_done_transfers`` mutates ``_sending_transfers``; the - # writer thread also appends to it, so guard the pop. - with self._sending_transfers_lock: - done_pushing, failed_pushing = self._pop_done_transfers( - self._sending_transfers - ) - # Remember failures until the final sibling WRITE completes. - self._send_failures.update(failed_pushing) - successful = { - req_id - for req_id in done_pushing - self._send_failures - if req_id in self._reqs_to_send or req_id in self._reqs_to_process - } - self._send_failures.difference_update(done_pushing | done_sending) - # Expired requests were already reported, even if their WRITEs finish later. - done_pushing = successful - for req_id in done_pushing: - self._reqs_to_send.pop(req_id, None) - self._reqs_to_process.discard(req_id) - done_sending.add(req_id) - - # Tell the writer to drop any state it still holds for any - # request that just finished (push completed) or expired - # (lease ran out without a D registration ever arriving). - for req_id in done_sending: - self._evict_finished_inbox.put(req_id) - if done_sending: - self._push_writer_wake.set() - - return results diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/scheduler.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/scheduler.py deleted file mode 100644 index 3da8e28a74..0000000000 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/scheduler.py +++ /dev/null @@ -1,12 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Backward-compatible re-export of NixlPullConnectorScheduler.""" - -from vllm.distributed.kv_transfer.kv_connector.v1.nixl.pull_scheduler import ( - NixlPullConnectorScheduler, -) - -# Backward compatibility: NixlConnectorScheduler is the pull-based scheduler. -NixlConnectorScheduler = NixlPullConnectorScheduler - -__all__ = ["NixlConnectorScheduler", "NixlPullConnectorScheduler"] diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/stats.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/stats.py deleted file mode 100644 index 645525bc6e..0000000000 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/stats.py +++ /dev/null @@ -1,328 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Stats and Prometheus metrics for the NIXL connector.""" - -import copy -from dataclasses import dataclass -from typing import TYPE_CHECKING, Any - -import numpy as np - -from vllm.config import VllmConfig -from vllm.distributed.kv_transfer.kv_connector.v1.metrics import ( - KVConnectorPromMetrics, - KVConnectorStats, - PromMetric, - PromMetricT, -) -from vllm.v1.metrics.utils import create_metric_per_engine - -if TYPE_CHECKING: - from vllm.distributed.nixl_utils import nixlXferTelemetry - - -@dataclass -class NixlKVConnectorStats(KVConnectorStats): - """Container for transfer performance metrics.""" - - def __post_init__(self): - if not self.data: - # Empty container init, no data is passed in. - self.reset() - - def reset(self): - # Must be serializable - self.data: dict[str, list[float | int]] = { - "transfer_duration": [], - "post_duration": [], - "bytes_transferred": [], - "num_descriptors": [], - "num_failed_transfers": [], - "num_failed_notifications": [], - "num_failed_handshakes": [], - "num_kv_expired_reqs": [], - "num_notifications_after_expiry": [], - } - - def record_transfer(self, res: "nixlXferTelemetry | None"): - # Keep metrics units consistent with rest of the code: time us->s - if res is None: - return - self.data["transfer_duration"].append(res.xferDuration / 1e6) - self.data["post_duration"].append(res.postDuration / 1e6) - self.data["bytes_transferred"].append(res.totalBytes) - self.data["num_descriptors"].append(res.descCount) - - def record_failed_transfer(self): - """Record a failed NIXL transfer operation.""" - self.data["num_failed_transfers"].append(1) - - def record_failed_notification(self): - """Record a failed NIXL notification (send_notif).""" - self.data["num_failed_notifications"].append(1) - - def record_failed_handshake(self): - """Record a failed NIXL handshake.""" - self.data["num_failed_handshakes"].append(1) - - def record_kv_expired_req(self): - """Record a request that had its KV blocks expire.""" - self.data["num_kv_expired_reqs"].append(1) - - def record_notification_after_expiry(self): - """Record a completion notification that arrived after its request expired.""" - self.data["num_notifications_after_expiry"].append(1) - - def clone_and_reset(self) -> "NixlKVConnectorStats": - old = copy.copy(self) - self.reset() - return old - - def is_empty(self) -> bool: - # Do not discard metrics update that are entirely failures related. - return ( - self.num_successful_transfers == 0 - and len(self.data["num_failed_transfers"]) == 0 - and len(self.data["num_failed_notifications"]) == 0 - and len(self.data["num_failed_handshakes"]) == 0 - and len(self.data["num_kv_expired_reqs"]) == 0 - and len(self.data["num_notifications_after_expiry"]) == 0 - ) - - def aggregate(self, other: KVConnectorStats) -> KVConnectorStats: - if not other.is_empty(): - for k, v in other.data.items(): - accumulator = self.data[k] - assert isinstance(accumulator, list) - accumulator.extend(v) - return self - - def reduce(self) -> dict[str, int | float]: - # Compute compact representative stats suitable for CLI logging. - # Failure counts are reported on every interval: transfer, handshake - # and notification failures are grouped as sporadic - # lower-transport-layer events, while KV expiry is reported separately - # as it is an actionable autoscaler signal rather than a transport - # issue. Notifications arriving after expiry are surfaced too, as the - # reader may have consumed KV blocks that were already reused. - failure_counts = { - "Num failed transfers": len(self.data["num_failed_transfers"]) - + len(self.data["num_failed_handshakes"]) - + len(self.data["num_failed_notifications"]), - "Num KV expired reqs": len(self.data["num_kv_expired_reqs"]), - "Num notifs after expiry": len(self.data["num_notifications_after_expiry"]), - } - if self.num_successful_transfers == 0: - # Timing / throughput stats only cover successful transfers. If - # all requests in the interval were unsuccessful, the failure - # counts above still surface what happened. - return { - "Num successful transfers": 0, - "Avg xfer time (ms)": 0, - "P90 xfer time (ms)": 0, - "Avg post time (ms)": 0, - "P90 post time (ms)": 0, - "Avg MB per transfer": 0, - "Throughput (MB/s)": 0, - "Avg number of descriptors": 0, - **failure_counts, - } - - xfer_time = np.asarray(self.data["transfer_duration"]) - post_time = np.asarray(self.data["post_duration"]) - # Convert to MB for CLI logging. - mb = np.asarray(self.data["bytes_transferred"]) / 2**20 - descs = np.asarray(self.data["num_descriptors"], dtype=np.uint32) - n = len(descs) - assert n == self.num_successful_transfers - - total_mb = mb.sum() - avg_mb = total_mb / n - - total_time_seconds = xfer_time.sum() - throughput_mb_s = total_mb / total_time_seconds - - return { - "Num successful transfers": n, - "Avg xfer time (ms)": round(xfer_time.mean().item() * 1e3, 3), - "P90 xfer time (ms)": round(np.percentile(xfer_time, 90).item() * 1e3, 3), - "Avg post time (ms)": round(post_time.mean().item() * 1e3, 3), - "P90 post time (ms)": round(np.percentile(post_time, 90).item() * 1e3, 3), - "Avg MB per transfer": round(avg_mb.item(), 3), - "Throughput (MB/s)": round(throughput_mb_s.item(), 3), - "Avg number of descriptors": round(descs.mean().item(), 1), - **failure_counts, - } - - @property - def num_successful_transfers(self) -> int: - return len(self.data["transfer_duration"]) - - -class NixlPromMetrics(KVConnectorPromMetrics): - def __init__( - self, - vllm_config: VllmConfig, - metric_types: dict[type[PromMetric], type[PromMetricT]], - labelnames: list[str], - per_engine_labelvalues: dict[int, list[object]], - ): - super().__init__(vllm_config, metric_types, labelnames, per_engine_labelvalues) - - buckets = [ - 0.001, - 0.005, - 0.01, - 0.025, - 0.05, - 0.075, - 0.1, - 0.2, - 0.3, - 0.5, - 0.75, - 1.0, - 5.0, - ] - nixl_histogram_xfer_time = self._histogram_cls( - name="vllm:nixl_xfer_time_seconds", - documentation="Histogram of transfer duration for NIXL KV Cache transfers.", - buckets=buckets[1:], - labelnames=labelnames, - ) - self.nixl_histogram_xfer_time = create_metric_per_engine( - nixl_histogram_xfer_time, self.per_engine_labelvalues - ) - nixl_histogram_post_time = self._histogram_cls( - name="vllm:nixl_post_time_seconds", - documentation="Histogram of transfer post time for NIXL KV" - " Cache transfers.", - buckets=buckets, - labelnames=labelnames, - ) - self.nixl_histogram_post_time = create_metric_per_engine( - nixl_histogram_post_time, self.per_engine_labelvalues - ) - # uniform 2kb to 16gb range - buckets = [2 ** (10 + i) for i in range(1, 25, 2)] - nixl_histogram_bytes_transferred = self._histogram_cls( - name="vllm:nixl_bytes_transferred", - documentation="Histogram of bytes transferred per NIXL KV Cache transfers.", - buckets=buckets, - labelnames=labelnames, - ) - self.nixl_histogram_bytes_transferred = create_metric_per_engine( - nixl_histogram_bytes_transferred, self.per_engine_labelvalues - ) - buckets = [ - 10, - 20, - 30, - 50, - 75, - 100, - 200, - 400, - 1000, - 2000, - 4000, - 10000, - 20000, - 50000, - ] - nixl_histogram_num_descriptors = self._histogram_cls( - name="vllm:nixl_num_descriptors", - documentation="Histogram of number of descriptors per NIXL" - " KV Cache transfers.", - buckets=buckets, - labelnames=labelnames, - ) - self.nixl_histogram_num_descriptors = create_metric_per_engine( - nixl_histogram_num_descriptors, self.per_engine_labelvalues - ) - counter_nixl_num_failed_transfers = self._counter_cls( - name="vllm:nixl_num_failed_transfers", - documentation="Number of failed NIXL KV Cache transfers, including" - " handshake and notification failures. NOTE: KV expiry is tracked" - " separately in vllm:nixl_num_kv_expired_reqs.", - labelnames=labelnames, - ) - self.counter_nixl_num_failed_transfers = create_metric_per_engine( - counter_nixl_num_failed_transfers, self.per_engine_labelvalues - ) - - counter_nixl_num_failed_notifications = self._counter_cls( - name="vllm:nixl_num_failed_notifications", - documentation="Number of failed NIXL KV Cache notifications. " - "Retained for compatibility; these failures are also included in " - "vllm:nixl_num_failed_transfers.", - labelnames=labelnames, - ) - self.counter_nixl_num_failed_notifications = create_metric_per_engine( - counter_nixl_num_failed_notifications, self.per_engine_labelvalues - ) - - counter_nixl_num_kv_expired_reqs = self._counter_cls( - name="vllm:nixl_num_kv_expired_reqs", - documentation="Number of requests that had their KV expire. " - "NOTE: This metric is tracked on the P instance.", - labelnames=labelnames, - ) - self.counter_nixl_num_kv_expired_reqs = create_metric_per_engine( - counter_nixl_num_kv_expired_reqs, self.per_engine_labelvalues - ) - - counter_nixl_num_notifications_after_expiry = self._counter_cls( - name="vllm:nixl_num_notifications_after_expiry", - documentation="Number of completion notifications for requests that " - "were no longer tracked, usually because their KV lease expired. The " - "KV blocks may have been reused before the transfer finished. Counted " - "per notification (one per remote rank), not per request.", - labelnames=labelnames, - ) - self.counter_nixl_num_notifications_after_expiry = create_metric_per_engine( - counter_nixl_num_notifications_after_expiry, self.per_engine_labelvalues - ) - - def observe(self, transfer_stats_data: dict[str, Any], engine_idx: int = 0): - for prom_obj, list_item_key in zip( - [ - self.nixl_histogram_xfer_time, - self.nixl_histogram_post_time, - self.nixl_histogram_bytes_transferred, - self.nixl_histogram_num_descriptors, - ], - [ - "transfer_duration", - "post_duration", - "bytes_transferred", - "num_descriptors", - ], - ): - for list_item in transfer_stats_data[list_item_key]: - prom_obj[engine_idx].observe(list_item) - for counter_obj, counter_item_keys in zip( - [ - self.counter_nixl_num_failed_transfers, - self.counter_nixl_num_failed_notifications, - self.counter_nixl_num_kv_expired_reqs, - self.counter_nixl_num_notifications_after_expiry, - ], - [ - # Transfer, handshake and notification failures are grouped: - # all are sporadic lower-transport-layer events. KV expiry is - # reported separately as it signals autoscaler behavior, not - # transport health. - ( - "num_failed_transfers", - "num_failed_handshakes", - "num_failed_notifications", - ), - ("num_failed_notifications",), - ("num_kv_expired_reqs",), - ("num_notifications_after_expiry",), - ], - ): - for counter_item_key in counter_item_keys: - for list_item in transfer_stats_data[counter_item_key]: - counter_obj[engine_idx].inc(list_item) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/tp_mapping.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/tp_mapping.py deleted file mode 100644 index 150fdabe35..0000000000 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/tp_mapping.py +++ /dev/null @@ -1,161 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""TP mapping computation for NIXL KV cache transfers.""" - -from __future__ import annotations - -from dataclasses import dataclass - -import numpy as np - -from vllm.distributed.kv_transfer.kv_connector.utils import ( - BlockIds, - TransferTopology, -) -from vllm.v1.kv_cache_interface import AttentionSpec, KVCacheSpec, MambaSpec - -# ====================================================================== -# Data structures -# ====================================================================== - - -@dataclass(frozen=True) -class ReadSpec: - """Specification for a single remote block read operation.""" - - remote_rank: int - local_block_ids: BlockIds - remote_block_ids: BlockIds - block_ids_by_region: bool = False - - -def _is_attention_spec(spec_type: type[KVCacheSpec]) -> bool: - return issubclass(spec_type, AttentionSpec) - - -def _is_ssm_spec(spec_type: type[KVCacheSpec]) -> bool: - return issubclass(spec_type, MambaSpec) - - -@dataclass(frozen=True) -class TPMapping: - """Complete local-to-remote TP mapping for one remote engine. - - Generated once per remote engine during handshake. - """ - - # Remote TP ranks that this local rank reads from, per group. - # Position = local piece index. - source_ranks_per_group: tuple[tuple[int, ...], ...] - - # Superset of all source ranks (union of all groups). - all_source_ranks: tuple[int, ...] - - # Maps each source rank to its FA head slot index. - rank_to_attention_slot: dict[int, int] - - # FA head offset factor for hetero-TP (D_TP > P_TP). - rank_offset_factor: int - - # Local ranks (in aggregate) that read from a given source rank. The producer frees - # a request's blocks only once that many notifications have come in. - local_consumers: int = 1 - - -# ====================================================================== -# TP mapping computation -# ====================================================================== - - -def compute_tp_mapping( - transfer_topology: TransferTopology, - remote_tp_size: int, - group_spec_types: tuple[type[KVCacheSpec], ...], - remote_dcp_size: int = 1, -) -> TPMapping: - """Build the complete local-to-remote TP mapping. - - Computes source ranks, head slot assignments, and the rank offset - factor in a single pass. - - DCP support is scoped to MLA only, with a side is either fully replicated or fully - sharded. DCP-branch reuses the same rank set used at handshake selection. - """ - tp_rank = transfer_topology.tp_rank - tp_size = transfer_topology.tp_size - total_num_kv_heads = transfer_topology.total_num_kv_heads - # --- Attention source ranks --- - if transfer_topology.is_mla or tp_size >= remote_tp_size: - if transfer_topology.is_mla and remote_dcp_size > 1: - attn_ranks = transfer_topology.dcp_source_ranks( - remote_tp_size, remote_dcp_size - ) - else: - # D (local TP) > P (remote TP): multiple local ranks read different chunks - # from *one* remote rank, corresponding to different kv heads. - # For MLA, we only need one remote since cache is duplicated. When - # P TP=k*TP k, this will spread mla ranks to read from remote k*tp_rank. - attn_ranks = [tp_rank * remote_tp_size // tp_size] - else: - # P (remote TP) > D (local TP): one local rank - # reads from multiple remote ranks. - # GQA dedup: when K < remote_tp_size, several remote ranks - # hold the same KV head. np.unique keeps only the first - # rank per unique head so we don't issue redundant reads. - abs_tp = remote_tp_size // tp_size - start = tp_rank * abs_tp - heads = np.arange(start, start + abs_tp) * total_num_kv_heads // remote_tp_size - _, unique_idx = np.unique(heads, return_index=True) - attn_ranks = (start + np.sort(unique_idx)).tolist() - - # --- SSM source ranks --- - has_ssm = any(_is_ssm_spec(t) for t in group_spec_types) - if has_ssm: - if tp_size < remote_tp_size: - abs_tp = remote_tp_size // tp_size - ssm_ranks = list(range(tp_rank * abs_tp, (tp_rank + 1) * abs_tp)) - else: - ssm_ranks = list(attn_ranks) - else: - ssm_ranks = [] - - all_ranks = sorted(set(attn_ranks) | set(ssm_ranks)) - - # --- Per-group ordered source ranks --- - source_ranks_per_group = tuple( - tuple(ssm_ranks) if _is_ssm_spec(t) else tuple(attn_ranks) - for t in group_spec_types - ) - - # --- Attention head slots --- - head_to_slot: dict[int, int] = {} - for i, r in enumerate(attn_ranks): - head_to_slot[r * total_num_kv_heads // remote_tp_size] = i - rank_to_attention_slot = { - r: head_to_slot.get(r * total_num_kv_heads // remote_tp_size, 0) - for r in all_ranks - } - - # --- Rank offset factor --- - if transfer_topology.is_mla or tp_size <= remote_tp_size: - # We don't index into remote for reading, no offset needed. - rank_offset_factor = 0 - elif tp_size > total_num_kv_heads: - local_head = tp_rank * total_num_kv_heads // tp_size - p_start = attn_ranks[0] * total_num_kv_heads // remote_tp_size - rank_offset_factor = local_head - p_start - else: - # D TP > P TP: we index into remote to read different heads depending on rank. - rank_offset_factor = tp_rank % (tp_size // remote_tp_size) - - local_consumers = transfer_topology.dcp_consumer_count( - remote_tp_size, remote_dcp_size - ) - - return TPMapping( - source_ranks_per_group=source_ranks_per_group, - all_source_ranks=tuple(all_ranks), - rank_to_attention_slot=rank_to_attention_slot, - rank_offset_factor=rank_offset_factor, - local_consumers=local_consumers, - ) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/utils.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/utils.py deleted file mode 100644 index 6ec0e73078..0000000000 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/utils.py +++ /dev/null @@ -1,67 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Shared constants, lazy imports and helpers for the NIXL connector.""" - -import contextlib -from collections.abc import Iterator -from typing import Any - -import regex as re -import zmq - -from vllm.platforms import current_platform -from vllm.utils.network_utils import make_zmq_socket -from vllm.v1.kv_cache_interface import KVCacheSpec, UniformTypeKVCacheSpecs - -# Supported platforms and types of kv transfer buffer. -# {device: tuple of supported kv buffer types} -_NIXL_SUPPORTED_DEVICE = { - "cuda": ( - "cuda", - "cpu", - ), - "tpu": ("cpu",), - "xpu": ( - "cpu", - "xpu", - ), - "cpu": ("cpu",), -} -# support for oot platform by providing mapping in current_platform -_NIXL_SUPPORTED_DEVICE.update(current_platform.get_nixl_supported_devices()) - - -# TODO: merge with vllm.utils.network_utils.zmq_socket_ctx -@contextlib.contextmanager -def zmq_ctx(socket_type: Any, addr: str) -> Iterator[zmq.Socket]: - """Context manager for a ZMQ socket.""" - if socket_type not in (zmq.ROUTER, zmq.REQ): - raise ValueError(f"Unexpected socket type: {socket_type}") - - ctx: zmq.Context | None = None - try: - ctx = zmq.Context() # type: ignore[attr-defined] - yield make_zmq_socket( - ctx=ctx, path=addr, socket_type=socket_type, bind=socket_type == zmq.ROUTER - ) - finally: - if ctx is not None: - ctx.destroy(linger=0) - - -def get_representative_spec_type(spec: KVCacheSpec) -> type[KVCacheSpec]: - if isinstance(spec, UniformTypeKVCacheSpecs): - # All inner specs are the same type; pick any. - inner = next(iter(spec.kv_cache_specs.values())) - return type(inner) - return type(spec) - - -# Trailing 8-hex randomization suffix appended by -# ``input_processor.assign_request_id`` as ``-{random_uuid():.8}``. -_RANDOM_SUFFIX_RE = re.compile(r"-[0-9a-f]{8}$", re.IGNORECASE) - - -def get_base_request_id(request_id: str) -> str: - """Strip the per-request ``-<8 hex>`` randomization suffix, if present.""" - return _RANDOM_SUFFIX_RE.sub("", request_id) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/worker.py deleted file mode 100644 index 66ad155bda..0000000000 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/worker.py +++ /dev/null @@ -1,13 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Backward-compatible re-export of NixlPullConnectorWorker.""" - -from vllm.distributed.kv_transfer.kv_connector.v1.nixl.pull_worker import ( - NixlPullConnectorWorker, -) - -# Backward compatibility: NixlConnectorWorker is the pull-based worker. -NixlConnectorWorker = NixlPullConnectorWorker - - -__all__ = ["NixlConnectorWorker", "NixlPullConnectorWorker"] diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/simple_cpu_offload_connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/simple_cpu_offload_connector.py deleted file mode 100644 index 49afa017f6..0000000000 --- a/vllm/distributed/kv_transfer/kv_connector/v1/simple_cpu_offload_connector.py +++ /dev/null @@ -1,386 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""SimpleCPUOffloadConnector: minimal CPU KV cache offloading.""" - -from collections.abc import Iterable -from typing import TYPE_CHECKING, Any - -import torch - -from vllm.config import VllmConfig -from vllm.distributed.kv_events import KVCacheEvent -from vllm.distributed.kv_transfer.kv_connector.v1.base import ( - KVConnectorBase_V1, - KVConnectorMetadata, - KVConnectorRole, - SupportsHMA, -) -from vllm.distributed.kv_transfer.kv_connector.v1.metrics import ( - KVConnectorPromMetrics, - KVConnectorStats, - PromMetric, - PromMetricT, -) -from vllm.logger import init_logger -from vllm.v1.core.sched.output import SchedulerOutput -from vllm.v1.metrics.cache_hit_source import CacheHitSource -from vllm.v1.outputs import KVConnectorOutput -from vllm.v1.simple_kv_offload.manager import ( - BoundaryStoreStats, - SimpleCPUOffloadScheduler, -) -from vllm.v1.simple_kv_offload.metadata import ( - SimpleCPUOffloadMetadata, -) -from vllm.v1.simple_kv_offload.metrics import ( - SimpleCPUOffloadPromMetrics, - SimpleCPUOffloadStats, -) -from vllm.v1.simple_kv_offload.worker import ( - SimpleCPUOffloadWorker, -) - -if TYPE_CHECKING: - from vllm.forward_context import ForwardContext - from vllm.v1.attention.backend import AttentionMetadata - from vllm.v1.core.block_pool import BlockPool - from vllm.v1.core.kv_cache_manager import KVCacheBlocks - from vllm.v1.kv_cache_interface import KVCacheConfig - from vllm.v1.request import Request - -logger = init_logger(__name__) - -# Default CPU capacity: 8 GB -DEFAULT_CPU_CAPACITY_BYTES = 8 * (1024**3) - -VALID_KV_OFFLOAD_BACKENDS = ("cpu", "disk") -# Keys that only apply to the disk backend, warned about under "cpu". -_DISK_ONLY_KEYS = ( - "disk_path", - "disk_capacity_bytes", - "disk_buffer_slots", - "use_page_cache", -) - - -class SimpleCPUOffloadConnector(KVConnectorBase_V1, SupportsHMA): - """CPU KV cache offloading with custom kernel transfers and BlockPool LRU.""" - - @property - def requires_kv_delivery(self) -> bool: - # Runs as kv_both, but is a best-effort cache: a dropped save is just a - # future cache miss, so opt out of the producer-role default. - return False - - def __init__( - self, - vllm_config: VllmConfig, - role: KVConnectorRole, - kv_cache_config: "KVCacheConfig", - ): - super().__init__(vllm_config, role, kv_cache_config) - - enable_prefix_caching = vllm_config.cache_config.enable_prefix_caching - extra_config = self._kv_transfer_config.kv_connector_extra_config or {} - - cpu_capacity_bytes = int( - extra_config.get("cpu_bytes_to_use", DEFAULT_CPU_CAPACITY_BYTES) - ) - # cpu_bytes_to_use is server-wide for compatibility; - # cpu_bytes_to_use_per_rank overrides for per-rank capacity. - world_size = vllm_config.parallel_config.world_size - cpu_capacity_per_rank = cpu_capacity_bytes // world_size - if "cpu_bytes_to_use_per_rank" in extra_config: - explicit = int(extra_config["cpu_bytes_to_use_per_rank"]) - if explicit != cpu_capacity_per_rank: - logger.warning( - "cpu_bytes_to_use_per_rank (%.2f GB) != " - "cpu_bytes_to_use/world_size (%.2f GB). Using per-rank value.", - explicit / (1024**3), - cpu_capacity_per_rank / (1024**3), - ) - cpu_capacity_per_rank = explicit - - # When set, num_cpu_blocks overrides automatic pool sizing from byte budgets - # (including cpu_bytes_to_use and cpu_bytes_to_use_per_rank), allowing ranks - # with different KV layouts (e.g. PP stages) to use the same pool size. - # Byte budgets still bound capacity. - num_cpu_blocks = extra_config.get("num_cpu_blocks") - if num_cpu_blocks is not None and ( - type(num_cpu_blocks) is not int or num_cpu_blocks <= 0 - ): - raise ValueError("num_cpu_blocks must be a positive integer") - - lazy_offload = bool(extra_config.get("lazy_offload", False)) - - kv_offload_backend = str(extra_config.get("kv_offload_backend", "cpu")) - if kv_offload_backend not in VALID_KV_OFFLOAD_BACKENDS: - raise ValueError( - f"Unknown kv_offload_backend {kv_offload_backend!r}; " - f"expected one of {VALID_KV_OFFLOAD_BACKENDS}" - ) - disk_mode = kv_offload_backend == "disk" - self._cache_hit_source = ( - CacheHitSource.DISK if disk_mode else CacheHitSource.HOST - ) - - disk_path = extra_config.get("disk_path", None) or None - disk_capacity_bytes = int( - extra_config.get("disk_capacity_bytes", 100 * (1024**3)) - ) - disk_buffer_slots = max(1, int(extra_config.get("disk_buffer_slots", 2))) - use_page_cache = bool(extra_config.get("use_page_cache", False)) - - if disk_mode: - if disk_path is None: - raise ValueError( - 'kv_offload_backend="disk" requires disk_path to be set.' - ) - if disk_capacity_bytes <= 0: - raise ValueError( - 'kv_offload_backend="disk" requires disk_capacity_bytes > 0.' - ) - else: - ignored = [k for k in _DISK_ONLY_KEYS if k in extra_config] - if ignored: - logger.warning( - 'kv_offload_backend is "cpu", ignoring disk-only config: %s. ' - 'Set kv_offload_backend="disk" to enable the disk backend.', - ", ".join(ignored), - ) - disk_path = None - - self.scheduler_manager: SimpleCPUOffloadScheduler | None = None - self.worker_handler: SimpleCPUOffloadWorker | None = None - - if not enable_prefix_caching: - logger.warning( - "Detected prefix caching disabled, disabling CPU offload " - "since it requires prefix caching." - ) - return - - logger.info( - "SimpleCPUOffloadConnector: role=%s, " - "per_rank=%.2f GB, world_size=%d, mode=%s, backend=%s, disk=%s", - role.name, - cpu_capacity_per_rank / (1024**3), - world_size, - "lazy" if lazy_offload else "eager", - kv_offload_backend, - disk_path or "none", - ) - if num_cpu_blocks is not None: - logger.info( - "Using explicit num_cpu_blocks=%d for offload pool sizing " - "instead of automatic sizing from byte budgets.", - num_cpu_blocks, - ) - - if role == KVConnectorRole.SCHEDULER: - from vllm.v1.core.kv_cache_utils import resolve_kv_cache_block_sizes - - assert kv_cache_config is not None - scheduler_block_size, hash_block_size = resolve_kv_cache_block_sizes( - kv_cache_config, vllm_config - ) - self.scheduler_manager = SimpleCPUOffloadScheduler( - vllm_config, - kv_cache_config, - cpu_capacity_per_rank, - scheduler_block_size=scheduler_block_size, - hash_block_size=hash_block_size, - lazy_offload=lazy_offload, - disk_capacity_bytes=disk_capacity_bytes if disk_mode else 0, - use_page_cache=use_page_cache if disk_mode else False, - num_cpu_blocks=num_cpu_blocks, - ) - elif role == KVConnectorRole.WORKER: - self.worker_handler = SimpleCPUOffloadWorker( - vllm_config, - kv_cache_config, - cpu_capacity_per_rank, - kv_offload_backend=kv_offload_backend, - disk_path=disk_path, - disk_capacity_bytes=disk_capacity_bytes, - disk_buffer_slots=disk_buffer_slots, - use_page_cache=use_page_cache, - num_cpu_blocks=num_cpu_blocks, - ) - - # --- Worker-side methods --- - - def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]) -> None: - if self.worker_handler is not None: - self.worker_handler.register_kv_caches(kv_caches) - - def bind_connector_metadata( - self, - connector_metadata: KVConnectorMetadata, - ) -> None: - super().bind_connector_metadata(connector_metadata) - if self.worker_handler is not None: - assert isinstance(connector_metadata, SimpleCPUOffloadMetadata) - self.worker_handler.bind_connector_metadata(connector_metadata) - - def clear_connector_metadata(self) -> None: - super().clear_connector_metadata() - if self.worker_handler is not None: - self.worker_handler.clear_connector_metadata() - - def handle_preemptions(self, kv_connector_metadata: KVConnectorMetadata) -> None: - if self.worker_handler is not None: - assert isinstance(kv_connector_metadata, SimpleCPUOffloadMetadata) - self.worker_handler.handle_preemptions(kv_connector_metadata) - - def start_load_kv(self, forward_context: "ForwardContext", **kwargs: Any) -> None: - if self.worker_handler is not None: - self.worker_handler.start_load_kv() - - def wait_for_layer_load(self, layer_name: str) -> None: - pass # Always load asynchronously, issued in start_load_kv() - - def save_kv_layer( - self, - layer_name: str, - kv_layer: torch.Tensor, - attn_metadata: "AttentionMetadata", - **kwargs: Any, - ) -> None: - pass # Always save asynchronously, issued in wait_for_save() - - def wait_for_save(self) -> None: - if self.worker_handler is not None: - self.worker_handler.wait_for_save() - - def get_finished( - self, - finished_req_ids: set[str], - ) -> tuple[set[str] | None, set[str] | None]: - if self.worker_handler is not None: - return self.worker_handler.get_finished(finished_req_ids) - return None, None - - def build_connector_worker_meta(self): - if self.worker_handler is not None: - return self.worker_handler.build_connector_worker_meta() - return None - - # --- Scheduler-side methods --- - - def bind_gpu_block_pool(self, gpu_block_pool: "BlockPool") -> None: - if self.scheduler_manager is not None: - self.scheduler_manager.bind_gpu_block_pool(gpu_block_pool) - - def get_num_new_matched_tokens( - self, - request: "Request", - num_computed_tokens: int, - ) -> tuple[int | None, bool]: - if self.scheduler_manager is not None: - return self.scheduler_manager.get_num_new_matched_tokens( - request, num_computed_tokens - ) - return 0, False - - def update_state_after_alloc( - self, - request: "Request", - blocks: "KVCacheBlocks", - num_external_tokens: int, - ) -> None: - if self.scheduler_manager is not None: - self.scheduler_manager.update_state_after_alloc( - request, blocks, num_external_tokens - ) - - def build_connector_meta( - self, - scheduler_output: SchedulerOutput, - ) -> KVConnectorMetadata: - if self.scheduler_manager is not None: - return self.scheduler_manager.build_connector_meta(scheduler_output) - return SimpleCPUOffloadMetadata() - - def update_connector_output( - self, - connector_output: KVConnectorOutput, - ) -> None: - if self.scheduler_manager is not None: - self.scheduler_manager.update_connector_output(connector_output) - - def request_finished( - self, - request: "Request", - block_ids: list[int], - ) -> tuple[bool, dict[str, Any] | None]: - if self.scheduler_manager is not None: - return self.scheduler_manager.request_finished(request, block_ids) - return False, None - - def request_finished_all_groups( - self, - request: "Request", - block_ids: tuple[list[int], ...], - ) -> tuple[bool, dict[str, Any] | None]: - if self.scheduler_manager is not None: - return self.scheduler_manager.request_finished_all_groups( - request, block_ids - ) - return False, None - - def has_pending_push_work(self) -> bool: - if self.scheduler_manager is not None: - return self.scheduler_manager.has_pending_stores() - return False - - def get_boundary_store_stats(self) -> BoundaryStoreStats | None: - """Return cumulative boundary-handoff store diagnostics.""" - if self.scheduler_manager is not None: - return self.scheduler_manager.get_boundary_store_stats() - return None - - def get_kv_connector_stats(self) -> KVConnectorStats | None: - # Worker-side callers may invoke this hook on connectors constructed - # via __new__ (as in test_worker.py), which lack scheduler_manager. - scheduler_manager: SimpleCPUOffloadScheduler | None = getattr( - self, "scheduler_manager", None - ) - if scheduler_manager is not None: - return scheduler_manager.get_stats() - return None - - @classmethod - def build_kv_connector_stats( - cls, data: dict[str, Any] | None = None - ) -> KVConnectorStats | None: - return ( - SimpleCPUOffloadStats(data=data) - if data is not None - else SimpleCPUOffloadStats() - ) - - @classmethod - def build_prom_metrics( - cls, - vllm_config: VllmConfig, - metric_types: dict[type[PromMetric], type[PromMetricT]], - labelnames: list[str], - per_engine_labelvalues: dict[int, list[object]], - ) -> KVConnectorPromMetrics: - return SimpleCPUOffloadPromMetrics( - vllm_config, metric_types, labelnames, per_engine_labelvalues - ) - - def take_events(self) -> Iterable[KVCacheEvent]: - if self.scheduler_manager is not None: - return self.scheduler_manager.take_events() - return [] - - # NOTE: Workers are not contacted. In-flight transfers drain naturally, - # and stale completions are ignored by the guarded - # SimpleCPUOffloadScheduler._process_store_event(). - def reset_cache(self) -> bool | None: - if self.scheduler_manager is not None: - return self.scheduler_manager.reset() - return None diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/ssm_conv_transfer_utils.py b/vllm/distributed/kv_transfer/kv_connector/v1/ssm_conv_transfer_utils.py deleted file mode 100644 index 04ce6a0b92..0000000000 --- a/vllm/distributed/kv_transfer/kv_connector/v1/ssm_conv_transfer_utils.py +++ /dev/null @@ -1,235 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Mamba conv-state sub-projection decomposition for NIXL transfer. - -With DS conv state layout (dim, state_len), sub-projections are -contiguous in memory. Each D rank reads its slices via separate -RDMA transfers — no P-side permutation needed. - -Supported model types: - - Mamba1: conv = [x], temporal = (intermediate_size, state_size) - - Mamba2: conv = [x, B, C], temporal = (num_heads, head_dim) - - GDN (Gated Delta Net): conv = [Q, K, V] (dim(Q)==dim(K)), - temporal = (num_v_heads, v_dim, k_dim) -""" - -import math -from dataclasses import dataclass - -import torch - -from vllm.model_executor.layers.mamba.mamba_utils import is_conv_state_dim_first -from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum -from vllm.v1.kv_cache_interface import MambaSpec - - -@dataclass(frozen=True) -class MambaConvSplitInfo: - """Per-rank byte sizes of the conv sub-projections. - - Used by both P and D sides for NIXL descriptor registration. - All fields are LOCAL to this engine's TP (already divided by TP size). - - DS memory layout within one page (contiguous): - Mamba1: |---- x ----| (single sub-projection, no decomposition) - Mamba2: |-- x --|- B -|- C -| (B == C) - GDN: |- Q -|- K -|-- V --| (dim(Q)==dim(K), V may differ) - """ - - conv_rows: int # conv_kernel - 1 (typically 3) - # Per-rank column counts per sub-projection: - # 1 entry for Mamba1, 3 for Mamba2/GDN. - local_proj_dims: tuple[int, ...] - conv_dtype_size: int # bytes per element (e.g. 2 for float16) - ssm_sizes: tuple[int, int] # (conv_state_bytes, ssm_state_bytes) - - @property - def local_conv_dim(self) -> int: - """Total conv columns per rank.""" - return sum(self.local_proj_dims) - - @property - def proj_bytes(self) -> tuple[int, ...]: - """Byte sizes of the sub-projections for one rank.""" - row_bytes = self.conv_rows * self.conv_dtype_size - return tuple(d * row_bytes for d in self.local_proj_dims) - - @property - def local_conv_offsets(self) -> list[tuple[int, int]]: - """(byte_offset, byte_size) of each sub-projection within this - engine's page. - - Used by both P and D for local descriptor registration. - """ - offsets: list[tuple[int, int]] = [] - offset = 0 - for size in self.proj_bytes: - offsets.append((offset, size)) - offset += size - return offsets - - def remote_conv_offsets( - self, local_rank_offset: int, tp_ratio: int - ) -> list[tuple[int, int]]: - """(byte_offset, byte_size) of this D rank's sub-projection slices - within one P page. - - Used by D side only, during remote descriptor registration. - - Args: - local_rank_offset: which slice this D rank reads. - tp_ratio: signed TP ratio. - >= 1: D_TP >= P_TP — P page is larger, D reads its slice. - < 0: P_TP > D_TP — P pages are smaller, D reads entire - P page. Local dims are scaled down by |tp_ratio| - to get P-sized offsets. - - """ - offsets: list[tuple[int, int]] = [] - if tp_ratio >= 1: - remote_base = 0 - for size in self.proj_bytes: - offsets.append((remote_base + local_rank_offset * size, size)) - remote_base += size * tp_ratio - else: - # NOTE (ZhanqiuHu): tp_ratio < 0 means P_TP > D_TP, so P pages - # are smaller than D's. Local dims are D-sized, but we need - # P-sized offsets. Scale down by |tp_ratio|. - abs_ratio = -tp_ratio - remote_base = 0 - for size in self.proj_bytes: - remote_size = size // abs_ratio - offsets.append((remote_base, remote_size)) - remote_base += remote_size - return offsets - - -def derive_mamba_conv_split( - mamba_spec: MambaSpec, - local_tp: int, -) -> MambaConvSplitInfo: - """Derive per-rank sub-projection byte sizes from a MambaSpec. - - Called once at init on both P and D. Decomposes the conv dimension - into its sub-projection parts based on the model type. - - Args: - mamba_spec: MambaSpec whose shapes are: - shapes[0] = conv state: (conv_dim_local, conv_rows) in DS layout. - shapes[1] = temporal state (model-specific shape). - local_tp: this engine's tensor-parallel size. - - Returns: - MambaConvSplitInfo with per-rank sub-projection dims, conv_rows, - conv_dtype_size, and ssm_sizes (conv_state_bytes, ssm_state_bytes). - - """ - _supported = ( - MambaAttentionBackendEnum.MAMBA1, - MambaAttentionBackendEnum.MAMBA2, - MambaAttentionBackendEnum.GDN_ATTN, - ) - if mamba_spec.mamba_type not in _supported: - raise NotImplementedError( - f"Conv transfer only supports Mamba1, Mamba2 and GDN models, " - f"got mamba_type={mamba_spec.mamba_type!r}." - ) - - conv_shape = mamba_spec.shapes[0] - assert len(conv_shape) == 2, f"Expected 2D conv state shape, got {conv_shape}" - - # NOTE (ZhanqiuHu): 3-read requires DS layout, which is already asserted - # in nixl worker __init__. Use it directly instead of heuristic detection. - assert is_conv_state_dim_first(), "3-read requires DS conv state layout" - local_conv_dim = conv_shape[0] # DS: (conv_dim_local, conv_rows) - conv_rows = conv_shape[1] - - conv_dtype_size = torch.tensor( - [], - dtype=mamba_spec.dtypes[0], # type: ignore[misc] - ).element_size() - ssm_dtype_size = torch.tensor( - [], - dtype=mamba_spec.dtypes[1], # type: ignore[misc] - ).element_size() - conv_state_bytes = torch.Size(mamba_spec.shapes[0]).numel() * conv_dtype_size - ssm_state_bytes = torch.Size(mamba_spec.shapes[1]).numel() * ssm_dtype_size - - local_proj_dims: tuple[int, ...] - if mamba_spec.mamba_type == MambaAttentionBackendEnum.MAMBA1: - # Mamba1 conv state holds only x (no B/C), so it's a single - # contiguous TP shard with no sub-projection decomposition. - temporal_shape = mamba_spec.shapes[1] - assert temporal_shape[0] == local_conv_dim, ( - f"Mamba1 temporal state dim ({temporal_shape[0]}) doesn't match " - f"conv dim ({local_conv_dim}); both should be " - f"intermediate_size/TP." - ) - local_proj_dims = (local_conv_dim,) - elif mamba_spec.mamba_type == MambaAttentionBackendEnum.MAMBA2: - # NOTE (ZhanqiuHu): intermediate_size (= global x dim) is not stored - # in MambaSpec, so we reconstruct it from the SSM temporal state shape: - # shapes[1] = (local_num_heads, head_dim), already divided by TP. - head_dim = mamba_spec.shapes[1][1] - local_num_heads = mamba_spec.shapes[1][0] - intermediate_size = local_num_heads * local_tp * head_dim - - # NOTE (ZhanqiuHu): global conv dim = intermediate_size + 2 * groups_ss, - # where groups_ss is the B (= C) dimension. B and C are always the same - # size, so we recover groups_ss from the remainder after subtracting x. - remainder = local_conv_dim * local_tp - intermediate_size - assert remainder > 0 and remainder % 2 == 0, ( - f"Conv dim ({local_conv_dim}*tp={local_tp}) doesn't decompose " - f"into intermediate_size={intermediate_size} + 2*groups_ss. " - f"remainder={remainder}" - ) - groups_ss = remainder // 2 - - # Divide by TP to get per-rank column counts. - x_local = intermediate_size // local_tp - b_local = groups_ss // local_tp - local_proj_dims = (x_local, b_local, b_local) - elif mamba_spec.mamba_type == MambaAttentionBackendEnum.GDN_ATTN: - # GDN: conv = [Q, K, V] where dim(Q) == dim(K) == key_dim. - # conv_dim = key_dim*2 + value_dim (all global, divided by TP). - # Temporal state shape is (num_v_heads/TP, head_v_dim, head_k_dim). - temporal_shape = mamba_spec.shapes[1] - num_v_heads_local = temporal_shape[0] - head_v_dim = temporal_shape[1] - value_dim_local = num_v_heads_local * head_v_dim - - remainder = local_conv_dim - value_dim_local - assert remainder > 0 and remainder % 2 == 0, ( - f"GDN conv dim ({local_conv_dim}) doesn't decompose into " - f"2*key_dim_local + value_dim_local={value_dim_local}. " - f"remainder={remainder}" - ) - key_dim_local = remainder // 2 - local_proj_dims = (key_dim_local, key_dim_local, value_dim_local) - else: - raise NotImplementedError( - f"Conv split not supported for mamba_type={mamba_spec.mamba_type!r}" - ) - - return MambaConvSplitInfo( - conv_rows=conv_rows, - local_proj_dims=local_proj_dims, - conv_dtype_size=conv_dtype_size, - ssm_sizes=(conv_state_bytes, ssm_state_bytes), - ) - - -def compute_physical_blocks_per_logical( - ssm_sizes: tuple[int, ...], block_len: int -) -> int: - """Derive _physical_blocks_per_logical_kv_block from remote metadata. - - The remote engine's ratio is not sent directly in the handshake, so we - reconstruct it: total mamba state per logical block / block_len. - - Args: - ssm_sizes: (conv_state_bytes, ssm_state_bytes) from NixlAgentMetadata. - block_len: the engine's block_len in bytes (from block_lens[0]). - - """ - return math.ceil((ssm_sizes[0] + ssm_sizes[1]) / block_len) diff --git a/vllm/distributed/weight_transfer/ipc_engine.py b/vllm/distributed/weight_transfer/ipc_engine.py deleted file mode 100644 index 13da117b99..0000000000 --- a/vllm/distributed/weight_transfer/ipc_engine.py +++ /dev/null @@ -1,459 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""IPC-based weight transfer engine using CUDA IPC for communication.""" - -import pickle -from dataclasses import asdict, dataclass -from typing import TYPE_CHECKING, Any, ClassVar, Self - -import pybase64 as base64 -import torch -from torch.multiprocessing.reductions import rebuild_cuda_tensor, reduce_tensor - -from vllm import envs -from vllm.config.weight_transfer import WeightTransferConfig -from vllm.distributed.weight_transfer.base import ( - TrainerInitInfo, - TrainerWeightTransferEngine, - VLLMWeightSyncClient, - WeightSource, - WeightTransferEngine, - WeightTransferInitInfo, - WeightTransferUpdateInfo, -) - -if TYPE_CHECKING: - from vllm.config import VllmConfig -from vllm.distributed.weight_transfer.packed_tensor import ( - DEFAULT_PACKED_BUFFER_SIZE_BYTES, - PackedBufferImporter, - packed_ipc_consumer, - packed_ipc_producer, -) - - -@dataclass -class IPCWeightTransferInitInfo(WeightTransferInitInfo): - """Worker-side init info for IPC weight transfer. No rendezvous needed. - - `packed` is a must-agree wire param: the trainer ships it here at the init - handshake so the worker decodes with the same setting the trainer encoded - with. The consumer rebuilds from the IPC handle + `tensor_sizes`, so it does - not need the buffer size (producer-only).""" - - packed: bool = False - - -@dataclass -class IPCTrainerInitInfo(TrainerInitInfo): - """Trainer-side init info for IPC weight transfer. No rendezvous needed; - `rank` (from `TrainerInitInfo`) identifies this trainer process — rank 0 - ships the merged IPC handles. All ranks still join the handle all-gather. - - `packed` / `packed_buffer_size_bytes` are the transfer's wire params. The - trainer propagates `packed` to the worker at `trainer_init` so the two sides - cannot disagree. `backend` is the factory dispatch key.""" - - backend: ClassVar[str] = "ipc" - packed: bool = False - packed_buffer_size_bytes: int = DEFAULT_PACKED_BUFFER_SIZE_BYTES - - -@dataclass -class IPCWeightTransferUpdateInfo(WeightTransferUpdateInfo): - """Per-round update info for the IPC weight transfer backend.""" - - names: list[str] - dtype_names: list[str] - shapes: list[list[int]] - ipc_handles: list[dict[str, tuple]] | dict[str, tuple] | None = None - """IPC handles mapping physical GPU UUID to rebuild_cuda_tensor args. - For non-packed mode: list of per-parameter handle dicts. - For packed mode: single handle dict for the packed buffer.""" - ipc_handles_pickled: str | None = None - """Base64-encoded pickled IPC handles, used for HTTP transport.""" - tensor_sizes: list[int] | None = None - """Per-parameter sizes in bytes within the packed buffer. - Required when packed=True, unused otherwise.""" - - def __post_init__(self): - if self.ipc_handles_pickled is not None: - if self.ipc_handles is not None: - raise ValueError( - "Cannot specify both `ipc_handles` and `ipc_handles_pickled`" - ) - - if not envs.VLLM_ALLOW_INSECURE_SERIALIZATION: - raise ValueError( - "Refusing to deserialize `ipc_handles_pickled` without " - "VLLM_ALLOW_INSECURE_SERIALIZATION=1" - ) - - self.ipc_handles = pickle.loads(base64.b64decode(self.ipc_handles_pickled)) - self.ipc_handles_pickled = None - - if self.ipc_handles is None: - raise ValueError( - "Either `ipc_handles` or `ipc_handles_pickled` must be provided" - ) - num_params = len(self.names) - if len(self.dtype_names) != num_params: - raise ValueError( - f"`dtype_names` should be of the same size as `names`: " - f"got {len(self.dtype_names)} and {len(self.names)}" - ) - if len(self.shapes) != num_params: - raise ValueError( - f"`shapes` should be of the same size as `names`: " - f"got {len(self.shapes)} and {len(self.names)}" - ) - # Unpacked transfers carry a per-parameter handle dict (a list); packed - # transfers carry a single dict for the whole buffer, so the list check - # only applies to the list case. - if isinstance(self.ipc_handles, list) and len(self.ipc_handles) != num_params: - raise ValueError( - f"`ipc_handles` should be of the same size as `names`: " - f"got {len(self.ipc_handles)} and {len(self.names)}" - ) - - -class IPCWeightTransferEngine( - WeightTransferEngine[IPCWeightTransferInitInfo, IPCWeightTransferUpdateInfo] -): - """Weight transfer engine using CUDA IPC between trainer and workers. - - This implementation uses CUDA IPC to transfer weights from the trainer (rank 0) - to all inference workers in a process group. IPC handles are used to share - memory between processes on the same node. - """ - - # Define backend-specific dataclass types - init_info_cls = IPCWeightTransferInitInfo - update_info_cls = IPCWeightTransferUpdateInfo - - def __init__( - self, - config: WeightTransferConfig, - vllm_config: "VllmConfig", - device: torch.device, - model: torch.nn.Module, - ) -> None: - super().__init__(config, vllm_config, device, model) - # Set from the trainer-supplied init info at the handshake; defaults are - # only for the (unreachable) receive-before-init case. - self.packed = False - # Shared across all chunks of every packed transfer this engine - # receives; see PackedBufferImporter for the refcount contract. - self._packed_importer = PackedBufferImporter() - - def init_transfer_engine(self, init_info: IPCWeightTransferInitInfo) -> None: - """Initialize the weight transfer mechanism. No data-plane rendezvous is - needed for IPC; this just records the trainer-supplied wire params so the - worker decodes exactly as the trainer encoded. - - Args: - init_info: IPC initialization info (carries `packed`). - - """ - self.packed = init_info.packed - - def start_weight_update(self) -> None: - """Initialize layerwise reloading for the incoming checkpoint weights.""" - from vllm.model_executor.model_loader.reload import ( - initialize_layerwise_reload, - ) - - initialize_layerwise_reload(self.model) - - def finish_weight_update(self) -> None: - """Finalize layerwise reloading after all weights have been received.""" - from vllm.model_executor.model_loader.reload import ( - finalize_layerwise_reload, - ) - - finalize_layerwise_reload(self.model, self.model_config) - # Every reduce_tensor call is a fresh export with its own refcount - # slot, so releasing once per update always balances this update's - # export and lets the trainer reclaim its staging buffer. Callers - # that skip finish are still covered by the replace-on-next-export - # path inside the importer. - self._packed_importer.close() - - def receive_weights(self, update_info: IPCWeightTransferUpdateInfo) -> None: - """Receive weights from the trainer via CUDA IPC handles and load them. - - Whether the transfer is packed is read from `self.packed`, set at the - init handshake from the trainer's init info, so it is guaranteed to - match how the trainer encoded. - - Args: - update_info: IPC update info containing parameter names, dtypes, shapes, - and IPC handles. Each IPC handle is a mapping between physical - GPU UUID and the rebuild_cuda_tensor args tuple. - - """ - # Use the worker's assigned device rather than the ambient current - # device: the receive path is no longer wrapped in - # `with torch.device(self.device)` by the caller, so the current device - # is not guaranteed to match self.device. The IPC tensors must be - # rebuilt on the device the model lives on. - device_index = self.device.index - - if self.packed: - if update_info.tensor_sizes is None: - raise ValueError("`tensor_sizes` is required when packed=True") - assert isinstance(update_info.ipc_handles, dict) - weights = packed_ipc_consumer( - ipc_handle=update_info.ipc_handles, - names=update_info.names, - shapes=update_info.shapes, - dtype_names=update_info.dtype_names, - tensor_sizes=update_info.tensor_sizes, - device_index=device_index, - importer=self._packed_importer, - ) - else: - assert isinstance(update_info.ipc_handles, list) - weights = [] - for name, ipc_handle in zip( - update_info.names, - update_info.ipc_handles, - ): - props = torch.cuda.get_device_properties(device_index) - physical_gpu_id = str(props.uuid) - - if physical_gpu_id not in ipc_handle: - raise ValueError( - f"IPC handle not found for GPU UUID " - f"{physical_gpu_id}. " - f"Available UUIDs: {list(ipc_handle.keys())}" - ) - - args = ipc_handle[physical_gpu_id] - list_args = list(args) - # Index 6 of the args from reduce_tensor is the device_index. - # We need to overwrite it with the receiver's device index. - list_args[6] = device_index - weight = rebuild_cuda_tensor(*list_args) - weights.append((name, weight)) - - from vllm.model_executor.model_loader.mtp_validation import ( - disable_mtp_completeness_check, - ) - - with disable_mtp_completeness_check(): - self.model.load_weights(weights) - - def shutdown(self) -> None: - self._packed_importer.close() - - -class IPCTrainerWeightTransferEngine(TrainerWeightTransferEngine[IPCTrainerInitInfo]): - """Trainer-side CUDA IPC weight transfer engine. - - Called on every trainer rank. For multi-rank (e.g. FSDP) trainers all ranks - iterate the source (materializing each tensor) and contribute to the - IPC-handle all-gather; only the sender (rank 0) ships the merged handles to - the inference side. IPC transfer is straight-line (no concurrent broadcast - like NCCL): `update_weights` *is* the transfer, and it rides the client, so - it no-ops on non-senders. - - `packed` / `packed_buffer_size_bytes` come from `IPCTrainerInitInfo`; the - sender propagates `packed` to the worker at `trainer_init`. - """ - - init_info_cls = IPCTrainerInitInfo - - def __init__( - self, - *, - client: VLLMWeightSyncClient, - source: WeightSource, - is_sender: bool = True, - packed: bool = False, - packed_buffer_size_bytes: int = DEFAULT_PACKED_BUFFER_SIZE_BYTES, - ) -> None: - super().__init__(client=client, source=source, is_sender=is_sender) - self.packed = packed - self.packed_buffer_size_bytes = packed_buffer_size_bytes - self.device_index = torch.accelerator.current_device_index() - self.gpu_uuid = str(torch.cuda.get_device_properties(self.device_index).uuid) - - @classmethod - def trainer_init( - cls, - init_info: IPCTrainerInitInfo, - *, - client: VLLMWeightSyncClient, - source: WeightSource | None = None, - ) -> Self: - if source is None: - raise ValueError("IPC trainer weight transfer requires a WeightSource.") - engine = cls( - client=client, - source=source, - is_sender=init_info.is_sender, - packed=init_info.packed, - packed_buffer_size_bytes=init_info.packed_buffer_size_bytes, - ) - # IPC needs no data-plane rendezvous. The sender ships the must-agree - # `packed` flag so the worker decodes exactly as this trainer encodes. - if engine.is_sender: - engine.client.init_weight_transfer_engine({"packed": init_info.packed}) - return engine - - def send_weights(self) -> None: - assert self.source is not None - source = self.source - if self.is_sender: - self.client.start_weight_update() - # Unpacked returns strong refs to its IPC-shared copies; they must stay - # alive across `finish_weight_update` too. - weight_refs = self._send(source) - if self.is_sender: - self.client.finish_weight_update() - self._post_send_sync() - del weight_refs - - # ---- data plane (runs on all ranks; only the sender ships) ---- - - def _send(self, source: WeightSource) -> list[torch.Tensor] | None: - if self.packed: - self._send_packed(source) - return None - return self._send_unpacked(source) - - def _all_gather_and_merge_handles( - self, - handles: list[dict[str, tuple]], - ) -> list[dict[str, tuple]]: - """All-gather and merge IPC handle dicts across ranks in one call. - - Each rank contributes a list of {gpu_uuid: ipc_args} dicts (one - per parameter or one per chunk). A single all_gather_object - collects every rank's full list, then the sender merges per-index so - each dict maps every GPU UUID to its args. - - The all-gather runs over the *default* process group; this assumes the - default group is exactly the set of colocated trainer ranks and that the - sender is a member. No-op (returns handles unchanged) when no distributed - group exists. - """ - if ( - not torch.distributed.is_initialized() - or torch.distributed.get_world_size() == 1 - ): - return handles - - world_size = torch.distributed.get_world_size() - gathered: list[list[dict[str, tuple]] | None] = [None] * world_size - torch.distributed.all_gather_object(gathered, handles) - torch.distributed.barrier() - torch.cuda.synchronize() - - if self.is_sender: - merged: list[dict[str, tuple]] = [] - for param_idx in range(len(handles)): - m: dict[str, tuple] = {} - for rank_handles in gathered: - if rank_handles is not None: - m.update(rank_handles[param_idx]) - merged.append(m) - return merged - return [{} for _ in handles] - - @staticmethod - def _post_send_sync() -> None: - """Barrier + ipc_collect after a send; no-op if single-GPU.""" - if ( - torch.distributed.is_initialized() - and torch.distributed.get_world_size() > 1 - ): - torch.distributed.barrier() - torch.cuda.ipc_collect() - - def _send_unpacked(self, source: WeightSource) -> list[torch.Tensor]: - """Iterate the source, build one IPC handle per param, all-gather the - handles across ranks, and (sender) ship them in one update call. - - Returns the strong refs to every contiguous copy. reduce_tensor's args - do NOT keep storage alive, and non-contiguous inputs allocate fresh - storage in .contiguous(); the caller must keep these alive until the - post-send barrier (past `finish`) so the consumer's IPC views stay valid. - """ - names: list[str] = [] - dtype_names: list[str] = [] - shapes: list[list[int]] = [] - ipc_handles: list[dict[str, tuple]] = [] - weight_refs: list[torch.Tensor] = [] - - for name, tensor in source: - # FSDP shards were gathered by the source; ensure contiguity here. - names.append(name) - dtype_names.append(str(tensor.dtype).split(".")[-1]) - shapes.append(list(tensor.shape)) - - weight = tensor.detach().contiguous() - weight_refs.append(weight) - _, ipc_args = reduce_tensor(weight) - ipc_handles.append({self.gpu_uuid: ipc_args}) - - ipc_handles = self._all_gather_and_merge_handles(ipc_handles) - self._do_send( - names=names, - dtype_names=dtype_names, - shapes=shapes, - ipc_handles=ipc_handles, - ) - return weight_refs - - def _send_packed(self, source: WeightSource) -> None: - """Send weights in bounded-memory chunks (packed mode).""" - for chunk in packed_ipc_producer( - iterator=iter(source), - gpu_uuid=self.gpu_uuid, - post_iter_func=lambda item: item[1], - buffer_size_bytes=self.packed_buffer_size_bytes, - ): - ipc_handle = self._all_gather_and_merge_handles([chunk.ipc_handle])[0] - self._do_send( - names=chunk.names, - dtype_names=chunk.dtype_names, - shapes=chunk.shapes, - ipc_handles=ipc_handle, - tensor_sizes=chunk.tensor_sizes, - ) - # Per-chunk barrier: the producer reuses a single IPC buffer across - # chunks, but only the sender waits for the consumers (via _do_send). - # Without syncing every rank here, non-sender ranks race ahead and - # overwrite their buffer while their colocated worker is still - # reading the current chunk, silently corrupting the transfer. - self._post_send_sync() - - def _do_send( - self, - names: list[str], - dtype_names: list[str], - shapes: list[list[int]], - ipc_handles: list[dict[str, tuple]] | dict[str, tuple], - tensor_sizes: list[int] | None = None, - ) -> None: - """Build one update payload and ship it via the client. Only the sender - ships (non-sender ranks already contributed to the handle all-gather). - - Emits raw `ipc_handles`; transports that cannot carry them natively - (HTTP/JSON) pickle them in their client (see `HTTPVLLMWeightSyncClient`). - """ - if not self.is_sender: - return - update_fields: dict[str, Any] = { - "names": names, - "dtype_names": dtype_names, - "shapes": shapes, - "ipc_handles": ipc_handles, - } - if tensor_sizes is not None: - update_fields["tensor_sizes"] = tensor_sizes - - update_info = IPCWeightTransferUpdateInfo(**update_fields) - self.client.update_weights(asdict(update_info)) diff --git a/vllm/distributed/weight_transfer/m2n_common.py b/vllm/distributed/weight_transfer/m2n_common.py deleted file mode 100644 index 13f7521408..0000000000 --- a/vllm/distributed/weight_transfer/m2n_common.py +++ /dev/null @@ -1,864 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Shared helpers for the NCCL M2N (`nccl_m2n`) weight transfer backend. - -M2N reshards a tensor between two disjoint meshes of ranks that live in one -communicator: the trainer occupies ranks `[0, T)` and the inference workers -`[T, T + N)`. This module holds everything both sides need — the optional -runtime import, layout descriptors, and the conversion to `nccl.m2n` types — -so the engine module stays about the transfer itself. -""" - -import hashlib -import json -from collections.abc import Mapping, Sequence -from dataclasses import dataclass -from typing import TYPE_CHECKING, Any, cast - -import torch - -from vllm.distributed.weight_transfer.base import ParamMeta -from vllm.utils.nccl import unpinned_nccl_env - -if TYPE_CHECKING: - from vllm.distributed.utils import StatelessProcessGroup - -# Placement code for a mesh axis that replicates. Any other (non-negative) code -# is the tensor dim that axis shards. Ints rather than `nccl.m2n` objects so -# layouts survive the JSON init handshake without a custom encoder. -REPLICATE = -1 - -# `NCCL_RESHARD_MAX_TENSOR_DIMS`; 4-D and higher are rejected by the library. -MAX_TENSOR_DIMS = 3 - -MESH_NDIMS = 2 - -# Destination mesh convention. Keeping these roles named avoids scattering -# positional axis assumptions through the worker-side planner. -DESTINATION_REPLICA_AXIS = 0 -DESTINATION_SHARD_AXIS = 1 - -# Conservative capacities accepted by the supported nccl_m2n algorithms. They -# are not exposed by its Python API yet; remove these mirrors once nccl_m2n -# provides communication-free plan validation. -MAX_SOURCE_SHARDS = 16 -MAX_DEST_SHARDS = 64 - -# Increment when the JSON init payload changes incompatibly. -M2N_WIRE_SCHEMA_VERSION = 2 - -SUPPORTED_DTYPES: frozenset[torch.dtype] = frozenset( - dtype - for dtype in ( - getattr(torch, name, None) - for name in ( - "int8", - "uint8", - "float8_e4m3fn", - "float8_e5m2", - "float16", - "bfloat16", - "int32", - "uint32", - "float32", - "int64", - "uint64", - "float64", - ) - ) - if dtype is not None -) - -_IMPORT_HINT = ( - "The nccl_m2n weight transfer backend requires the `nccl-extensions` " - "package (and its `nccl4py` dependency), which is not installed by vLLM. " - "See https://github.com/NVIDIA/nccl-extensions for build " - "and install instructions. It needs NCCL 2.30.5 or newer." -) - - -def prepare_m2n_local_runtime(m2n: Any, max_cta: int | None) -> tuple[Any, int, Any]: - """Prepare rank-local resources before any rank enters NCCL init.""" - import nccl.core as nccl - - device = torch.accelerator.current_device_index() - if not isinstance(device, int) or isinstance(device, bool) or device < 0: - raise RuntimeError(f"invalid current accelerator device index: {device!r}") - handle = m2n.Handle.create(m2n.Config(max_cta=max_cta)) - return nccl, device, handle - - -def create_m2n_communicator( - nccl: Any, - group: "StatelessProcessGroup", - unique_id_bytes: bytes | None, -) -> Any: - """Create the M2N data-plane communicator through NCCL4Py.""" - if unique_id_bytes is None: - envelope: dict[str, bytes | str | None] | None = None - if group.rank == 0: - try: - envelope = { - "unique_id": bytes(nccl.get_unique_id().as_bytes), - "error": None, - } - except Exception as exc: - envelope = { - "unique_id": None, - "error": f"{type(exc).__name__}: {exc}", - } - envelope = group.broadcast_obj(envelope, src=0) - if not isinstance(envelope, Mapping) or set(envelope) != {"unique_id", "error"}: - raise RuntimeError("invalid NCCL4Py unique-id envelope") - error = envelope["error"] - received_unique_id = envelope["unique_id"] - if error is not None: - if not isinstance(error, str) or not error: - raise RuntimeError("invalid NCCL4Py unique-id error") - raise RuntimeError(f"NCCL4Py unique-id generation failed: {error}") - if not isinstance(received_unique_id, bytes): - raise RuntimeError("NCCL4Py unique-id broadcast did not return bytes") - unique_id_bytes = received_unique_id - - unique_id = nccl.UniqueId.from_bytes(unique_id_bytes) - with unpinned_nccl_env(): - return nccl.Communicator.init( - nranks=group.world_size, - rank=group.rank, - unique_id=unique_id, - ) - - -def import_m2n() -> Any: - """Import `nccl.m2n` lazily, with an actionable error when it is missing. - - Deferred so that importing vLLM — or any other weight transfer backend — - never requires the m2n runtime to be present. - """ - try: - import nccl.m2n as m2n - except ImportError as e: - raise ImportError(f"{_IMPORT_HINT} (import failed: {e})") from e - return m2n - - -@dataclass(frozen=True) -class M2NMesh: - """One side's rank topology — pure topology, no tensor placement. - - Mirrors `ncclMesh_t`: a 2-axis mesh owning the contiguous rank interval - `[start_rank, start_rank + dims[0] * dims[1])`. There is no 1-D mesh; a - single-axis topology is spelled with a second axis of size 1. - - A model may use a different factorization for each tensor. For example, - dense weights can use a DP x TP mesh while expert weights use EDP x EP. - """ - - dims: tuple[int, int] - start_rank: int - - def __post_init__(self) -> None: - object.__setattr__(self, "dims", tuple(self.dims)) - if len(self.dims) != MESH_NDIMS or any( - not isinstance(dim, int) or isinstance(dim, bool) or dim <= 0 - for dim in self.dims - ): - raise ValueError(f"mesh dims must be {MESH_NDIMS} positive ints") - if ( - not isinstance(self.start_rank, int) - or isinstance(self.start_rank, bool) - or self.start_rank < 0 - ): - raise ValueError( - f"mesh start_rank must be non-negative, got {self.start_rank}" - ) - - @property - def size(self) -> int: - return self.dims[0] * self.dims[1] - - -# A tensor's placement over its side's mesh: one code per mesh axis -# (`REPLICATE`, or the tensor dim that axis shards), or `REPLICATED` for a -# tensor every rank holds in full. -Placements = tuple[int, int] -REPLICATED: Placements | None = None - - -@dataclass(frozen=True) -class M2NLayout: - """A tensor's mesh and placement on one side of a transfer.""" - - mesh: M2NMesh - placements: Placements | None - - def __post_init__(self) -> None: - if not isinstance(self.mesh, M2NMesh): - raise TypeError("M2N layout mesh must be an M2NMesh") - if self.placements is not None: - object.__setattr__(self, "placements", tuple(self.placements)) - - -def check_placements(placements: Placements, context: str = "placements") -> None: - """Reject placement pairs nccl_m2n cannot express.""" - if len(placements) != MESH_NDIMS: - raise ValueError(f"{context} must have {MESH_NDIMS} entries") - if any(not isinstance(code, int) or isinstance(code, bool) for code in placements): - raise TypeError(f"{context} must contain integer placement codes") - invalid = [code for code in placements if code < REPLICATE] - if invalid: - raise ValueError( - f"{context} contains invalid placement code {invalid[0]}; valid " - f"codes are {REPLICATE} (Replicate) and non-negative tensor dimensions" - ) - num_sharded = sum(code != REPLICATE for code in placements) - if num_sharded == 0: - raise ValueError( - f"{context}: a fully replicated tensor is carried as REPLICATED, " - "not as two REPLICATE axes" - ) - if num_sharded == MESH_NDIMS: - raise ValueError( - f"{context}: nccl_m2n needs one REPLICATE mesh axis, but " - f"{placements} shards both" - ) - - -def resolve_layout( - mesh: M2NMesh, - placements: Placements | None, - context: str = "placements", -) -> tuple[M2NMesh, Placements]: - """Pair one tensor's placement with the mesh m2n should see for it. - - A replicated tensor needs a size-1 mesh axis to carry a no-op shard, since - m2n has no `{REPLICATE, REPLICATE}`. It is therefore described over the - *same rank interval* re-factored as `(size, 1)`. That re-factoring is sound - precisely because replication is order-independent — every rank holds the - whole tensor, so it does not matter that `(a, b)` and `(size, 1)` walk the - interval in a different order. A sharded tensor keeps its side's own - factorization, where rank order decides who owns which shard. - """ - if placements is None: - return M2NMesh((mesh.size, 1), mesh.start_rank), (REPLICATE, 0) - check_placements(placements, context) - return mesh, placements - - -def validate_layout( - mesh: M2NMesh, - placements: Placements, - shape: Sequence[int], - context: str, -) -> tuple[int, ...]: - """Validate one resolved layout and return its required local shape.""" - local_shape = list(shape) - for axis, tensor_dim in enumerate(placements): - if tensor_dim == REPLICATE: - continue - if tensor_dim >= len(shape): - raise ValueError( - f"{context} shards tensor dim {tensor_dim}, but parameter has " - f"rank {len(shape)}" - ) - factor = mesh.dims[axis] - if shape[tensor_dim] % factor: - raise ValueError( - f"{context} shards dim {tensor_dim} (size {shape[tensor_dim]}) " - f"over {factor} ranks, which does not divide evenly" - ) - local_shape[tensor_dim] //= factor - return tuple(local_shape) - - -def expected_local_shape(meta: "M2NParamMeta") -> tuple[int, ...]: - """Validate a parameter's source layout and return its local shape.""" - mesh, placements = resolve_layout( - meta.source_layout.mesh, - meta.source_layout.placements, - f"parameter '{meta.name}' source placements", - ) - return validate_layout( - mesh, placements, meta.shape, f"parameter '{meta.name}' source layout" - ) - - -def check_transferable(name: str, dtype: torch.dtype, shape: Sequence[int]) -> None: - """Reject parameters nccl_m2n cannot transfer.""" - if dtype not in SUPPORTED_DTYPES: - raise ValueError( - f"parameter '{name}' has dtype {dtype}, which nccl_m2n does not " - f"support. Supported: {sorted(str(d) for d in SUPPORTED_DTYPES)}" - ) - if not 1 <= len(shape) <= MAX_TENSOR_DIMS: - raise ValueError( - f"parameter '{name}' has rank {len(shape)}; nccl_m2n supports " - f"rank 1..{MAX_TENSOR_DIMS}" - ) - - -def _shard_spec(mesh: M2NMesh, placements: Placements) -> tuple[int, int]: - return next( - (code, mesh.dims[axis]) - for axis, code in enumerate(placements) - if code != REPLICATE - ) - - -def _max_partition_overlaps( - extent: int, - partitions: int, - other_partitions: int, -) -> int: - """Return the maximum other partitions intersected by any one partition.""" - if extent == 0: - return 0 - width = extent // partitions - other_width = extent // other_partitions - max_overlaps = 0 - for index in range(partitions): - start = index * width - end = start + width - first_other = start // other_width - last_other = (end - 1) // other_width - max_overlaps = max(max_overlaps, last_other - first_other + 1) - return max_overlaps - - -def check_plan_limits( - src: tuple[M2NMesh, Placements], - dst: tuple[M2NMesh, Placements], - name: str, - shape: Sequence[int], -) -> None: - """Reject plans whose per-shard fan-in or fan-out exceeds M2N capacity.""" - src_dim, src_shards = _shard_spec(*src) - dst_dim, dst_shards = _shard_spec(*dst) - if any(extent == 0 for extent in shape): - max_sources = max_targets = 0 - elif src_dim != dst_dim: - max_sources = src_shards - max_targets = dst_shards - else: - extent = shape[src_dim] - max_sources = _max_partition_overlaps(extent, dst_shards, src_shards) - max_targets = _max_partition_overlaps(extent, src_shards, dst_shards) - - if max_sources > MAX_SOURCE_SHARDS: - raise ValueError( - f"parameter '{name}' would feed up to {max_sources} source shards " - f"into one destination shard, over nccl_m2n MAX_SOURCES=" - f"{MAX_SOURCE_SHARDS}" - ) - if max_targets > MAX_DEST_SHARDS: - raise ValueError( - f"parameter '{name}' would fan one source shard out to up to " - f"{max_targets} destination shards, over nccl_m2n " - f"MAX_TARGETS={MAX_DEST_SHARDS}" - ) - - -def validate_transfer_plan( - metas: Sequence["M2NParamMeta"], - dst_mesh: M2NMesh, - dst_placements: Sequence[Placements | None], -) -> None: - """Validate the complete ordered source-to-destination transfer plan.""" - if len(dst_placements) != len(metas): - raise ValueError( - f"destination plan has {len(dst_placements)} parameters, expected " - f"{len(metas)}" - ) - for meta, placement in zip(metas, dst_placements): - check_transferable(meta.name, meta.dtype, meta.shape) - src = resolve_layout( - meta.source_layout.mesh, - meta.source_layout.placements, - f"parameter '{meta.name}' source placements", - ) - validate_layout(*src, meta.shape, f"parameter '{meta.name}' source layout") - dst = resolve_layout( - dst_mesh, - placement, - f"parameter '{meta.name}' destination placements", - ) - validate_layout(*dst, meta.shape, f"parameter '{meta.name}' destination layout") - check_plan_limits(src, dst, meta.name, meta.shape) - - -@dataclass(frozen=True) -class M2NParamMeta(ParamMeta): - """`ParamMeta` extended with how the trainer places this tensor. - - The base class carries only name / dtype / full shape, which is not enough - to plan a reshard. `source_layout` carries both the per-parameter mesh and - its placement on that mesh. - """ - - source_layout: M2NLayout - - def __post_init__(self) -> None: - if not isinstance(self.source_layout, M2NLayout): - raise TypeError("source_layout must be an M2NLayout") - - -@dataclass(frozen=True) -class M2NWireParam: - """JSON-safe source metadata for one parameter in the init handshake.""" - - name: str - dtype_name: str - shape: tuple[int, ...] - src_mesh_dims: tuple[int, int] - src_placements: Placements | None - - def __post_init__(self) -> None: - shape = tuple(self.shape) - dims = tuple(self.src_mesh_dims) - placements = None if self.src_placements is None else tuple(self.src_placements) - object.__setattr__(self, "shape", shape) - object.__setattr__(self, "src_mesh_dims", dims) - object.__setattr__(self, "src_placements", placements) - if not isinstance(self.name, str) or not self.name: - raise ValueError("M2N wire parameter name must not be empty") - if not isinstance(self.dtype_name, str) or not self.dtype_name: - raise ValueError(f"parameter '{self.name}' has an empty wire dtype name") - if not shape or any( - not isinstance(dim, int) or isinstance(dim, bool) or dim <= 0 - for dim in shape - ): - raise ValueError( - f"parameter '{self.name}' shape must contain positive integers" - ) - M2NMesh(cast(tuple[int, int], dims), 0) - if placements is not None: - check_placements( - cast(Placements, placements), - f"parameter '{self.name}' source placements", - ) - - @classmethod - def from_dict(cls, value: Mapping[str, Any]) -> "M2NWireParam": - """Parse one strict wire record from its JSON-decoded mapping.""" - fields = { - "name", - "dtype_name", - "shape", - "src_mesh_dims", - "src_placements", - } - missing = fields - value.keys() - extra = value.keys() - fields - if missing or extra: - details = [] - if missing: - details.append(f"missing {sorted(missing)}") - if extra: - details.append(f"unexpected {sorted(extra)}") - raise ValueError(f"invalid M2N wire parameter: {', '.join(details)}") - return cls(**value) - - def to_dict(self) -> dict[str, Any]: - """Serialize this record using only JSON-compatible values.""" - return { - "name": self.name, - "dtype_name": self.dtype_name, - "shape": list(self.shape), - "src_mesh_dims": list(self.src_mesh_dims), - "src_placements": ( - None if self.src_placements is None else list(self.src_placements) - ), - } - - -def source_plan_digest(params: Sequence[M2NWireParam]) -> str: - """Return a rank-independent digest of the ordered source wire plan.""" - payload = json.dumps( - { - "schema_version": M2N_WIRE_SCHEMA_VERSION, - "params": [param.to_dict() for param in params], - }, - sort_keys=True, - separators=(",", ":"), - ).encode() - return hashlib.sha256(payload).hexdigest() - - -def complete_plan_digest( - source_digest: str, - dst_mesh: M2NMesh, - dst_placements: Sequence[Placements | None], -) -> str: - """Return a digest of the agreed source and destination plan.""" - if not isinstance(source_digest, str) or not source_digest: - raise ValueError("source digest must be a non-empty string") - payload = json.dumps( - { - "schema_version": M2N_WIRE_SCHEMA_VERSION, - "source_digest": source_digest, - "destination_mesh": { - "dims": list(dst_mesh.dims), - "start_rank": dst_mesh.start_rank, - }, - "destination_placements": [ - None if placement is None else list(placement) - for placement in dst_placements - ], - }, - sort_keys=True, - separators=(",", ":"), - ).encode() - return hashlib.sha256(payload).hexdigest() - - -def check_source_plan_agreement( - group: "StatelessProcessGroup", - params: Sequence[M2NWireParam], - expected_digest: str | None = None, - local_error: str | None = None, -) -> str: - """Verify every participant entered with the same source plan.""" - prepared_error = local_error - digest: str | None = None - declared: str | None = expected_digest - if prepared_error is None: - try: - digest = source_plan_digest(params) - declared = digest if expected_digest is None else expected_digest - if not isinstance(declared, str) or not declared: - raise ValueError("declared source digest must be a non-empty string") - except Exception as exc: - prepared_error = f"{type(exc).__name__}: {exc}" - elif not isinstance(prepared_error, str) or not prepared_error: - prepared_error = "TypeError: local source error must be a non-empty string" - - gathered = group.all_gather_obj( - { - "phase": "source", - "digest": digest, - "declared_digest": declared, - "error": prepared_error, - } - ) - errors: list[str] = [] - expected_fields = {"phase", "digest", "declared_digest", "error"} - if len(gathered) != group.world_size: - errors.append( - "control-plane all-gather returned " - f"{len(gathered)} records for world size {group.world_size}" - ) - agreed: list[tuple[int, str, str]] = [] - for rank, item in enumerate(gathered): - if not isinstance(item, Mapping) or set(item) != expected_fields: - errors.append(f"rank {rank}: invalid source preflight envelope") - continue - if item["phase"] != "source": - errors.append(f"rank {rank}: invalid source preflight phase") - continue - error = item["error"] - if error is not None: - if not isinstance(error, str) or not error: - errors.append(f"rank {rank}: invalid source preflight error") - else: - errors.append(f"rank {rank}: {error}") - continue - computed = item["digest"] - claimed = item["declared_digest"] - if ( - not isinstance(computed, str) - or not computed - or not isinstance(claimed, str) - or not claimed - ): - errors.append(f"rank {rank}: invalid source digest") - continue - agreed.append((rank, computed, claimed)) - if errors: - raise RuntimeError( - "nccl_m2n source preflight rejected before NCCL init: " + "; ".join(errors) - ) - - canonical = agreed[0][1] - for rank, computed, claimed in agreed: - if computed != claimed: - errors.append( - f"rank {rank}: computed digest {computed} does not match " - f"declared digest {claimed}" - ) - if computed != canonical: - errors.append(f"rank {rank}: source digest disagrees with rank 0") - if errors: - raise RuntimeError( - "nccl_m2n source preflight rejected before NCCL init: " + "; ".join(errors) - ) - return canonical - - -def check_complete_plan_agreement( - group: "StatelessProcessGroup", - source_digest: str, - dst_mesh: M2NMesh, - dst_placements: Sequence[Placements | None], - local_error: str | None = None, -) -> str: - """Require every rank to accept the same complete transfer plan.""" - prepared_error = local_error - digest: str | None = None - if prepared_error is None: - try: - digest = complete_plan_digest(source_digest, dst_mesh, dst_placements) - except Exception as exc: - prepared_error = f"{type(exc).__name__}: {exc}" - elif not isinstance(prepared_error, str) or not prepared_error: - prepared_error = "TypeError: local plan error must be a non-empty string" - - gathered = group.all_gather_obj( - {"phase": "complete_plan", "digest": digest, "error": prepared_error} - ) - errors: list[str] = [] - valid: list[tuple[int, str]] = [] - expected_fields = {"phase", "digest", "error"} - if len(gathered) != group.world_size: - errors.append( - "control-plane all-gather returned " - f"{len(gathered)} records for world size {group.world_size}" - ) - for rank, item in enumerate(gathered): - if ( - not isinstance(item, Mapping) - or set(item) != expected_fields - or item.get("phase") != "complete_plan" - ): - errors.append(f"rank {rank}: invalid complete-plan envelope") - continue - error = item["error"] - if error is not None: - if not isinstance(error, str) or not error: - errors.append(f"rank {rank}: invalid complete-plan error") - else: - errors.append(f"rank {rank}: {error}") - continue - rank_digest = item["digest"] - if not isinstance(rank_digest, str) or not rank_digest: - errors.append(f"rank {rank}: invalid complete-plan digest") - continue - valid.append((rank, rank_digest)) - if not errors: - canonical = valid[0][1] - for rank, rank_digest in valid[1:]: - if rank_digest != canonical: - errors.append( - f"rank {rank}: complete-plan digest disagrees with rank 0" - ) - if errors: - raise RuntimeError( - "nccl_m2n complete-plan preflight rejected before first reshard: " - + "; ".join(errors) - ) - return valid[0][1] - - -def check_runtime_ready_agreement( - group: "StatelessProcessGroup", local_error: str | None -) -> None: - """Require every rank to prepare local resources before NCCL init.""" - prepared_error = local_error - if prepared_error is not None and ( - not isinstance(prepared_error, str) or not prepared_error - ): - prepared_error = "TypeError: local runtime error must be a non-empty string" - gathered = group.all_gather_obj({"phase": "runtime_ready", "error": prepared_error}) - errors: list[str] = [] - if len(gathered) != group.world_size: - errors.append( - "control-plane all-gather returned " - f"{len(gathered)} records for world size {group.world_size}" - ) - for rank, item in enumerate(gathered): - if ( - not isinstance(item, Mapping) - or set(item) != {"phase", "error"} - or item.get("phase") != "runtime_ready" - ): - errors.append(f"rank {rank}: invalid runtime readiness envelope") - continue - error = item.get("error") - if error is not None: - if not isinstance(error, str) or not error: - errors.append(f"rank {rank}: invalid runtime readiness error") - else: - errors.append(f"rank {rank}: {error}") - if errors: - raise RuntimeError( - "nccl_m2n local runtime preflight failed before NCCL init: " - + "; ".join(errors) - ) - - -def check_data_plane_agreement( - group: "StatelessProcessGroup", - unique_id_bytes: bytes | None, - local_error: str | None = None, -) -> None: - """Agree on the NCCL rendezvous identity before NCCL init.""" - prepared_error = local_error - uid_digest: str | None = None - if prepared_error is None: - try: - if unique_id_bytes is not None: - if not isinstance(unique_id_bytes, bytes): - raise TypeError("NCCL unique id must decode to bytes") - uid_digest = hashlib.sha256(unique_id_bytes).hexdigest() - except Exception as exc: - prepared_error = f"{type(exc).__name__}: {exc}" - elif not isinstance(prepared_error, str) or not prepared_error: - prepared_error = "TypeError: local data-plane error must be non-empty" - - gathered = group.all_gather_obj( - { - "phase": "data_plane", - "mode": "uid" if unique_id_bytes is not None else "tcp", - "uid_digest": uid_digest, - "error": prepared_error, - } - ) - expected_fields = { - "phase", - "mode", - "uid_digest", - "error", - } - errors: list[str] = [] - valid: list[tuple[int, str, str | None]] = [] - if len(gathered) != group.world_size: - errors.append( - "control-plane all-gather returned " - f"{len(gathered)} records for world size {group.world_size}" - ) - for rank, item in enumerate(gathered): - if ( - not isinstance(item, Mapping) - or set(item) != expected_fields - or item.get("phase") != "data_plane" - ): - errors.append(f"rank {rank}: invalid data-plane preflight envelope") - continue - error = item["error"] - if error is not None: - if not isinstance(error, str) or not error: - errors.append(f"rank {rank}: invalid data-plane preflight error") - else: - errors.append(f"rank {rank}: {error}") - continue - mode = item["mode"] - uid = item["uid_digest"] - if mode not in {"tcp", "uid"} or (mode == "uid") != isinstance(uid, str): - errors.append(f"rank {rank}: invalid data-plane identity") - continue - valid.append((rank, mode, uid)) - if not errors: - _, mode, uid = valid[0] - for rank, other_mode, other_uid in valid[1:]: - if (other_mode, other_uid) != (mode, uid): - errors.append(f"rank {rank}: NCCL data-plane identity disagrees") - if errors: - raise RuntimeError( - "nccl_m2n data-plane preflight rejected before NCCL init: " - + "; ".join(errors) - ) - - -def validate_local_buffer(m2n: Any, meta: M2NParamMeta, buffer: Any) -> None: - """Check source dtype/shape while delegating buffer validity to M2N.""" - # A raw pointer carries no inspectable metadata. The source owns its - # correctness; _send supplies the declared dtype and local shape to M2N. - if isinstance(buffer, int): - return - - mesh, placements = resolve_layout( - meta.source_layout.mesh, meta.source_layout.placements - ) - actual = m2n.DistTensor( - buffer, - mesh=to_mesh(m2n, mesh), - placements=to_placements(m2n, placements), - ) - declared = m2n.DistTensor( - None, - local_shape=actual.local_shape, - dtype=meta.dtype, - mesh=to_mesh(m2n, mesh), - placements=to_placements(m2n, placements), - ) - if actual.dtype != declared.dtype: - raise ValueError( - f"parameter '{meta.name}' source returned dtype {actual.dtype}, " - f"but its metadata declared {declared.dtype}" - ) - - expected = expected_local_shape(meta) - if actual.local_shape != expected: - raise ValueError( - f"parameter '{meta.name}' source returned local shape " - f"{actual.local_shape}, but layout {meta.source_layout} implies " - f"{expected} from global shape {meta.shape}" - ) - - -def to_mesh(m2n: Any, mesh: M2NMesh) -> Any: - return m2n.Mesh(mesh.dims, start_rank=mesh.start_rank) - - -def to_placements(m2n: Any, placements: Placements) -> list[Any]: - return [ - m2n.Replicate() if code == REPLICATE else m2n.Shard(code) for code in placements - ] - - -def publish_destination_placements( - comm: Any, - device: torch.device, - stream: torch.cuda.Stream, - first_worker_rank: int, - placements: "Sequence[Placements | None] | None", - num_parameters: int, -) -> list[Placements | None]: - """Share the worker-side destination plan with every rank in the group. - - The trainer must issue each reshard with the same destination the workers - use, but once destinations are per-parameter they depend on the inference - model, which only the workers can see. The first worker publishes them here, - over the shared NCCL communicator; trainer ranks pass `None` and receive - them, and the other workers pass their own so a disagreement is caught - rather than deadlocking later. - - Only placements travel: the destination *mesh* is still derived from the - rank split, identically on both sides. The plan is published on the data - plane before any M2N reshard collective. - """ - replicated_sentinel = REPLICATE - 1 - encoded = torch.full( - (num_parameters, MESH_NDIMS), - replicated_sentinel, - dtype=torch.int8, - device=device, - ) - if comm.rank == first_worker_rank: - if placements is None or len(placements) != num_parameters: - raise ValueError( - f"publishing rank needs {num_parameters} destination placements" - ) - rows = [ - [replicated_sentinel] * MESH_NDIMS if placement is REPLICATED else placement - for placement in placements - ] - if rows: - encoded.copy_(torch.tensor(rows, dtype=torch.int8, device=device)) - - comm.broadcast(encoded, encoded, root=first_worker_rank, stream=stream) - return [ - REPLICATED - if all(code == replicated_sentinel for code in row) - else cast(Placements, tuple(row)) - for row in encoded.tolist() - ] diff --git a/vllm/distributed/weight_transfer/m2n_engine.py b/vllm/distributed/weight_transfer/m2n_engine.py deleted file mode 100644 index 66dbb6ba24..0000000000 --- a/vllm/distributed/weight_transfer/m2n_engine.py +++ /dev/null @@ -1,599 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Inference-side weight transfer engine built on NCCL M2N (`nccl_m2n`). - -The trainer and the inference workers share one NCCL communicator: trainer ranks -occupy `[0, T)`, workers `[T, T + N)`. Each parameter is moved with a single -`nccl.m2n.reshard`, which redistributes it from the trainer's layout (FSDP / EP -/ arbitrary DTensor sharding) to the inference layout — so the trainer sends its -local shards and never all-gathers a full tensor, which is what the broadcast -NCCL backend forces it to do. - -When an incoming checkpoint parameter maps directly to a live vLLM parameter, -M2N reshards into that worker's local model storage. Parameters whose names or -layouts cannot be resolved receive a full tensor and fall back to -`load_weights`, preserving the existing backend's loading behavior. - -Both meshes participate in every reshard, in the same order, so this backend has -the same concurrency shape as the broadcast NCCL backend: the worker must be -inside `receive_weights` while the trainer is sending. Driving that from the -trainer is the trainer engine's job. -""" - -from collections.abc import Iterator, Mapping -from dataclasses import dataclass, field -from typing import TYPE_CHECKING, Any, cast - -import torch - -from vllm.config.weight_transfer import WeightTransferConfig -from vllm.distributed.weight_transfer.base import ( - WeightTransferEngine, - WeightTransferInitInfo, - WeightTransferUpdateInfo, -) -from vllm.distributed.weight_transfer.m2n_common import ( - DESTINATION_SHARD_AXIS, - M2N_WIRE_SCHEMA_VERSION, - MESH_NDIMS, - M2NLayout, - M2NMesh, - M2NParamMeta, - M2NWireParam, - Placements, - check_complete_plan_agreement, - check_data_plane_agreement, - check_runtime_ready_agreement, - check_source_plan_agreement, - check_transferable, - create_m2n_communicator, - expected_local_shape, - import_m2n, - prepare_m2n_local_runtime, - publish_destination_placements, - resolve_layout, - source_plan_digest, - to_mesh, - to_placements, - validate_transfer_plan, -) -from vllm.distributed.weight_transfer.m2n_layout import ( - M2NDestination, - resolve_parameter_destinations, -) -from vllm.distributed.weight_transfer.nccl_common import ( - NCCLWeightTransferInitInfo, - decode_nccl_unique_id, - worker_init_metadata_group, -) -from vllm.logger import init_logger - -if TYPE_CHECKING: - from vllm.config import VllmConfig - -logger = init_logger(__name__) - -__all__ = [ - "M2NWeightTransferInitInfo", - "M2NWeightTransferUpdateInfo", - "M2NWeightTransferEngine", -] - - -# --------------------------------------------------------------------------- -# Wire types -# --------------------------------------------------------------------------- - - -@dataclass(kw_only=True) -class M2NWeightTransferInitInfo(WeightTransferInitInfo): - """Worker-side init info: the rendezvous plus the full transfer plan. - - Layouts are static for the whole run, so they ride the one-time init - handshake and the per-round update info stays a list of names. Everything is - plain JSON so the HTTP control plane carries it unchanged. - """ - - schema_version: int - master_address: str - master_port: int - rank_offset: int - """First worker rank, i.e. the number of trainer ranks.""" - world_size: int - """Trainer ranks + all inference workers.""" - dst_mesh_dims: list[int] - """The inference mesh, starting at `rank_offset`. Declared rather than - derived so both sides describe the destination identically.""" - source_digest: str - """SHA-256 of the ordered, versioned source parameter plan.""" - params: list[dict[str, Any]] - """Ordered per-parameter source metadata and layouts.""" - nccl_unique_id_b64: str | None = field(default=None, repr=False) - max_cta: int | None = None - - def __post_init__(self) -> None: - if not isinstance(self.master_address, str) or not self.master_address: - raise ValueError("`master_address` must name the metadata rendezvous") - if ( - not isinstance(self.master_port, int) - or isinstance(self.master_port, bool) - or not 0 < self.master_port < 65536 - ): - raise ValueError("`master_port` must be a valid metadata port") - if ( - not isinstance(self.rank_offset, int) - or isinstance(self.rank_offset, bool) - or not isinstance(self.world_size, int) - or isinstance(self.world_size, bool) - or self.rank_offset < 1 - or self.rank_offset >= self.world_size - ): - raise ValueError( - f"`rank_offset` ({self.rank_offset}) must leave at least one " - f"trainer rank and one worker in world_size {self.world_size}" - ) - num_workers = self.world_size - self.rank_offset - dst = tuple(self.dst_mesh_dims) - try: - dst_mesh = M2NMesh(cast(tuple[int, int], dst), self.rank_offset) - except (TypeError, ValueError) as exc: - raise ValueError(f"invalid `dst_mesh_dims` {dst}: {exc}") from exc - if dst_mesh.size != num_workers: - raise ValueError( - f"`dst_mesh_dims` {dst} must be {MESH_NDIMS} dims covering the " - f"{num_workers} inference workers" - ) - - def parse_wire_params(self) -> tuple[M2NWireParam, ...]: - """Parse and authenticate the ordered source plan.""" - if self.schema_version != M2N_WIRE_SCHEMA_VERSION: - raise ValueError( - "unsupported nccl_m2n init schema version " - f"{self.schema_version}; expected {M2N_WIRE_SCHEMA_VERSION}" - ) - if not isinstance(self.params, list): - raise TypeError("`params` must be a list of wire parameter mappings") - if not isinstance(self.source_digest, str) or not self.source_digest: - raise ValueError("`source_digest` must be a non-empty string") - wire_params: list[M2NWireParam] = [] - names: set[str] = set() - for index, value in enumerate(self.params): - if not isinstance(value, Mapping): - raise TypeError( - f"`params[{index}]` must be a mapping, got {type(value).__name__}" - ) - param = M2NWireParam.from_dict(value) - if param.name in names: - raise ValueError( - f"nccl_m2n init params contain duplicate parameter '{param.name}'" - ) - names.add(param.name) - wire_params.append(param) - parsed = tuple(wire_params) - computed_digest = source_plan_digest(parsed) - if self.source_digest != computed_digest: - raise ValueError( - "nccl_m2n init source digest does not match params: " - f"computed {computed_digest}, declared {self.source_digest}" - ) - return parsed - - @property - def nccl_unique_id_bytes(self) -> bytes | None: - if self.nccl_unique_id_b64 is None: - return None - return decode_nccl_unique_id( - master_address=None, - master_port=None, - nccl_unique_id_b64=self.nccl_unique_id_b64, - ctx="M2NWeightTransferInitInfo data plane", - ) - - -@dataclass -class M2NWeightTransferUpdateInfo(WeightTransferUpdateInfo): - """Per-round update info: which parameters this chunk carries, in order. - - Shapes, dtypes and layouts were fixed at init, so a round only needs to say - what is coming and in what order — both sides must issue their reshards in - exactly that order. - """ - - names: list[str] - - -# --------------------------------------------------------------------------- -# Worker engine -# --------------------------------------------------------------------------- - - -class M2NWeightTransferEngine( - WeightTransferEngine[M2NWeightTransferInitInfo, M2NWeightTransferUpdateInfo] -): - """Inference-side engine: receives each parameter with one reshard. - - Resolvable parameters are resharded from the trainer layout directly into - each worker's live local model storage. Unresolvable parameters receive a - full tensor and use `load_weights`. In both cases, the trainer sends its - local shards without materializing a full tensor. - """ - - init_info_cls = M2NWeightTransferInitInfo - update_info_cls = M2NWeightTransferUpdateInfo - # The destination plan is resolved against the model this engine was built - # for, so it cannot be retargeted at a draft model mid-flight. - supports_draft_weight_update = False - - def __init__( - self, - config: WeightTransferConfig, - vllm_config: "VllmConfig", - device: torch.device, - model: torch.nn.Module, - ) -> None: - super().__init__(config, vllm_config, device, model) - self.model_update_group: Any = None - self._m2n: Any = None - self._handle: Any = None - self._metas: list[M2NParamMeta] = [] - self._dst_mesh: M2NMesh | None = None - self._parameter_destinations: list[M2NDestination] = [] - self._index: dict[str, int] = {} - self._uses_load_weights = False - - def init_transfer_engine(self, init_info: M2NWeightTransferInitInfo) -> None: - """Agree on source and local readiness before constructing NCCL.""" - if self._handle is not None or self.model_update_group is not None: - raise RuntimeError( - "nccl_m2n weight transfer is already initialized; call " - "shutdown_weight_transfer_engine() before initializing again" - ) - rendezvous_info = NCCLWeightTransferInitInfo( - master_address=init_info.master_address, - master_port=init_info.master_port, - rank_offset=init_info.rank_offset, - world_size=init_info.world_size, - ) - metadata_group = worker_init_metadata_group( - rendezvous_info, self.parallel_config - ) - - wire_params: tuple[M2NWireParam, ...] = () - source_error: str | None = None - try: - self._m2n = import_m2n() - wire_params = init_info.parse_wire_params() - self._prepare_source_plan(wire_params, init_info.rank_offset) - except Exception as exc: - source_error = f"{type(exc).__name__}: {exc}" - self._metas = [] - check_source_plan_agreement( - metadata_group, - wire_params, - expected_digest=init_info.source_digest, - local_error=source_error, - ) - - destination_error: str | None = None - try: - self._prepare_destination_plan(init_info) - except Exception as exc: - destination_error = f"{type(exc).__name__}: {exc}" - self._parameter_destinations = [] - - unique_id_bytes: bytes | None = None - data_plane_error: str | None = None - try: - unique_id_bytes = init_info.nccl_unique_id_bytes - except Exception as exc: - data_plane_error = f"{type(exc).__name__}: {exc}" - check_data_plane_agreement( - metadata_group, - unique_id_bytes, - data_plane_error, - ) - - handle = None - group = None - try: - runtime_error = destination_error - nccl = None - device = None - if runtime_error is None: - try: - nccl, device, handle = prepare_m2n_local_runtime( - self._m2n, init_info.max_cta - ) - except Exception as exc: - runtime_error = f"{type(exc).__name__}: {exc}" - check_runtime_ready_agreement(metadata_group, runtime_error) - assert nccl is not None and device is not None and handle is not None - group = create_m2n_communicator(nccl, metadata_group, unique_id_bytes) - - # Keep the merged destination-placement publication unchanged: source - # agreement is independent of worker-side destination selection. - mine = [ - destination.placements for destination in self._parameter_destinations - ] - agreed = publish_destination_placements( - group, - self.device, - torch.cuda.current_stream(), - init_info.rank_offset, - mine, - len(self._metas), - ) - complete_plan_error: str | None = None - try: - if agreed != mine: - mismatched = next( - meta.name - for meta, ours, theirs in zip(self._metas, mine, agreed) - if ours != theirs - ) - raise RuntimeError( - "inference workers resolved different nccl_m2n " - f"destinations (first mismatch: '{mismatched}'); all " - "workers must run the same model and parallel config" - ) - assert self._dst_mesh is not None - validate_transfer_plan(self._metas, self._dst_mesh, agreed) - except Exception as exc: - complete_plan_error = f"{type(exc).__name__}: {exc}" - assert self._dst_mesh is not None - check_complete_plan_agreement( - metadata_group, - init_info.source_digest, - self._dst_mesh, - agreed, - complete_plan_error, - ) - except BaseException: - if group is not None: - try: - # Transport cleanup only; reload validity belongs to the - # model-update lifecycle, not this backend. - group.abort() - except BaseException: - logger.exception( - "failed to abort partially initialized nccl_m2n communicator" - ) - if handle is not None: - try: - handle.destroy() - except BaseException: - logger.exception( - "failed to destroy partially initialized nccl_m2n handle" - ) - raise - - self._handle = handle - self.model_update_group = group - - def _prepare_source_plan( - self, - wire_params: tuple[M2NWireParam, ...], - num_trainer_ranks: int, - ) -> None: - """Validate worker-visible source metadata without constructing NCCL.""" - self._metas = [] - for param in wire_params: - dtype = getattr(torch, param.dtype_name, None) - if not isinstance(dtype, torch.dtype): - raise ValueError( - f"parameter '{param.name}' has invalid dtype name " - f"'{param.dtype_name}'; expected the name of a torch.dtype" - ) - check_transferable(param.name, dtype, param.shape) - layout = M2NLayout(M2NMesh(param.src_mesh_dims, 0), param.src_placements) - if layout.mesh.size != num_trainer_ranks: - raise ValueError( - f"parameter '{param.name}' source mesh covers " - f"{layout.mesh.size} ranks, but there are " - f"{num_trainer_ranks} trainer ranks" - ) - meta = M2NParamMeta(param.name, dtype, param.shape, layout) - expected_local_shape(meta) - self._metas.append(meta) - - def _prepare_destination_plan(self, init_info: M2NWeightTransferInitInfo) -> None: - """Resolve the merged-#51520 in-place/full-fallback destination plan.""" - # The inference topology, as declared by the trainer. This is the mesh - # a sharded destination is placed over; a replicated one is described - # by the alternate descriptor `resolve_layout` derives from it, which - # covers the same ranks but is not this mesh. - self._dst_mesh = M2NMesh( - cast(tuple[int, int], tuple(init_info.dst_mesh_dims)), - init_info.rank_offset, - ) - parallel_config = self.parallel_config - shard_axis_size = self._dst_mesh.dims[DESTINATION_SHARD_AXIS] - quantization = getattr(self.model_config, "quantization", None) - # A quantized weight loader may dequantize / transpose / requantize, and - # under PP a rank's local shape is not the checkpoint shape split over - # the shard axis. Both break shape-derived resolution, so take the - # fallback. - allow_direct = ( - quantization is None and parallel_config.pipeline_parallel_size == 1 - ) - # Match every incoming checkpoint parameter to its rank-local target. - # Resolvable parameters can be resharded directly into live model - # storage; the rest receive a full tensor for `load_weights`. - self._parameter_destinations = resolve_parameter_destinations( - self.model, - [m.name for m in self._metas], - [m.dtype for m in self._metas], - [m.shape for m in self._metas], - num_workers=self._dst_mesh.size, - shard_axis_size=shard_axis_size, - allow_direct=allow_direct, - ) - validate_transfer_plan( - self._metas, - self._dst_mesh, - [destination.placements for destination in self._parameter_destinations], - ) - # Update requests carry names only, so cache their plan indices. Track - # whether any fallback entry requires the `load_weights` lifecycle. - self._index = {meta.name: i for i, meta in enumerate(self._metas)} - self._uses_load_weights = any( - not destination.direct for destination in self._parameter_destinations - ) - - def start_weight_update(self) -> None: - """Set up layerwise reloading, but only if some parameter needs it. - - Directly-resharded parameters are written in place and never go through - `load_weights`, so a plan with no fallback entries has nothing to - reload. - """ - if not self._uses_load_weights: - return - from vllm.model_executor.model_loader.reload import ( - initialize_layerwise_reload, - ) - - initialize_layerwise_reload(self.model) - - def finish_weight_update(self) -> None: - """Finalize layerwise reloading when the plan uses fallback entries.""" - if not self._uses_load_weights: - return - from vllm.model_executor.model_loader.reload import finalize_layerwise_reload - - finalize_layerwise_reload(self.model, self.model_config) - - def receive_weights(self, update_info: M2NWeightTransferUpdateInfo) -> None: - """Receive each requested parameter using its initialization-time plan.""" - if self._handle is None or self.model_update_group is None: - raise RuntimeError( - "nccl_m2n weight transfer not initialized. " - "Call init_transfer_engine() first." - ) - if not isinstance(update_info.names, list) or any( - not isinstance(name, str) for name in update_info.names - ): - raise TypeError("nccl_m2n update names must be a list of strings") - - requested = [] - for name in update_info.names: - index = self._index.get(name) - if index is None: - raise ValueError( - f"parameter '{name}' was not declared at init; the " - "trainer must send the same parameter set it announced" - ) - requested.append((name, index)) - - from vllm.model_executor.model_loader.mtp_validation import ( - disable_mtp_completeness_check, - ) - - stream = torch.cuda.current_stream() - - # Reshard lazily so one `load_weights` invocation sees the complete - # fallback sequence without staging every full tensor at once. Direct - # destinations still execute in request order as the iterator advances. - def reshard_requested_weights() -> Iterator[tuple[str, torch.Tensor]]: - for name, index in requested: - meta = self._metas[index] - destination = self._parameter_destinations[index] - buffer = destination.tensor - if buffer is None: - buffer = torch.empty( - meta.shape, dtype=meta.dtype, device=self.device - ) - - self._reshard( - self.model_update_group, - stream, - meta, - destination.placements, - buffer, - ) - - if destination.tensor is None: - # `load_weights` reads on the host stream, so the transfer - # has to have landed before it runs. - stream.synchronize() - yield name, buffer - - has_fallback = any( - self._parameter_destinations[index].tensor is None for _, index in requested - ) - with disable_mtp_completeness_check(): - received_weights = reshard_requested_weights() - if has_fallback: - # Some model loaders keep invocation-local state to combine - # related checkpoint tensors, so preserve one iterable scope. - self.model.load_weights(received_weights) - else: - # The iterator performs the reshards. With nothing to yield to - # `load_weights`, exhaust it here to execute direct transfers. - for _ in received_weights: - pass - - def _reshard( - self, - comm: Any, - stream: torch.cuda.Stream, - src_meta: M2NParamMeta, - dst_placements: Placements | None, - dst_buffer: torch.Tensor, - ) -> None: - """Reshard one parameter from its trainer layout into a worker buffer. - - `dst_placements` describes the buffer's logical placement over the - worker mesh; `dst_buffer` is this rank's physical storage. - """ - assert self._dst_mesh is not None - m2n = self._m2n - src_mesh, resolved_src_placements = resolve_layout( - src_meta.source_layout.mesh, src_meta.source_layout.placements - ) - dst_mesh, resolved_dst_placements = resolve_layout( - self._dst_mesh, dst_placements - ) - m2n.reshard( - None, - dst_buffer, - comm, - stream, - src_mesh=to_mesh(m2n, src_mesh), - src_placements=to_placements(m2n, resolved_src_placements), - dst_mesh=to_mesh(m2n, dst_mesh), - dst_placements=to_placements(m2n, resolved_dst_placements), - handle=self._handle, - ) - - def shutdown(self) -> None: - """Finish pending GPU work and release M2N communication state.""" - handle, self._handle = self._handle, None - group, self.model_update_group = self.model_update_group, None - self._m2n = None - self._metas = [] - self._dst_mesh = None - self._parameter_destinations = [] - self._index = {} - self._uses_load_weights = False - - first_error: BaseException | None = None - - def attempt(action: Any, message: str) -> None: - nonlocal first_error - try: - action() - except BaseException as exc: - if first_error is None: - first_error = exc - logger.exception(message) - - if handle is not None or group is not None: - # M2N does not synchronize caller streams on finalize. - attempt(torch.accelerator.synchronize, "failed to synchronize nccl_m2n") - if handle is not None: - attempt(handle.destroy, "failed to destroy nccl_m2n handle") - if group is not None: - attempt(group.abort, "failed to abort nccl_m2n communicator") - if first_error is not None: - raise first_error diff --git a/vllm/distributed/weight_transfer/m2n_layout.py b/vllm/distributed/weight_transfer/m2n_layout.py deleted file mode 100644 index 3296ef63fb..0000000000 --- a/vllm/distributed/weight_transfer/m2n_layout.py +++ /dev/null @@ -1,261 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Destination-layout resolution for the NCCL M2N weight transfer backend. - -For every incoming checkpoint parameter the worker needs a destination buffer -plus its placement over the inference mesh. Two outcomes are possible: - -* **direct** — when the model is not quantized, the parameter maps 1:1 onto a - live vLLM parameter and its loader is a small, known-safe copy or tensor- - parallel loader. The loader's declared input/output dimension determines the - placement; shapes are validation, not an inference mechanism. The reshard - writes straight into the live parameter, so each rank receives only its own - shard and nothing is copied afterwards. -* **fallback** — anything else. The reshard delivers the whole tensor to every - rank and `load_weights` does the sharding, exactly as the broadcast NCCL - backend does. Fused parameters (`qkv_proj`, `gate_up_proj`, MoE `w13`/`w2`) - take this path: the checkpoint name does not name a vLLM parameter, so there - is nothing to resolve against. - -Correctness never depends on a parameter resolving — the fallback is always -available and is the same path the existing backend uses. -""" - -import inspect -import math -from collections.abc import Sequence -from dataclasses import dataclass -from functools import partial -from typing import cast - -import torch - -from vllm.distributed.weight_transfer.m2n_common import ( - DESTINATION_REPLICA_AXIS, - DESTINATION_SHARD_AXIS, - MESH_NDIMS, - REPLICATE, - REPLICATED, - Placements, -) -from vllm.logger import init_logger - -logger = init_logger(__name__) - -_DIRECT_WEIGHT_LOADING_BLOCKLIST = { - "vllm.model_executor.models.gpt2.GPT2Model", -} - - -def _module_type_name(module: torch.nn.Module) -> str: - cls = type(module) - return f"{cls.__module__}.{cls.__name__}" - - -def _destination_placements(shard_dim: int) -> Placements: - """Place one tensor shard on the configured destination mesh axis.""" - placements = [REPLICATE] * MESH_NDIMS - placements[DESTINATION_REPLICA_AXIS] = REPLICATE - placements[DESTINATION_SHARD_AXIS] = shard_dim - return cast(Placements, tuple(placements)) - - -@dataclass -class M2NDestination: - """Where one checkpoint parameter lands on this worker.""" - - name: str - placements: Placements | None - """Placement over the inference mesh, or `REPLICATED` for the fallback.""" - tensor: torch.Tensor | None - """The live parameter view to reshard into, or None for the fallback path - (the engine allocates a full replica per round and calls `load_weights`).""" - - @property - def direct(self) -> bool: - return self.tensor is not None - - -def _base_loader_shard_dim( - param: torch.nn.Parameter, - shard_axis_size: int, -) -> int | None: - """Return a known-safe loader's declared shard dim, or reject it. - - Loader identity is deliberately fail-closed. A missing loader is not - treated as ``default_weight_loader`` because a model-level ``load_weights`` - implementation may transform the tensor before it reaches that default. - Wrappers and partials are also rejected: recognizing their wrapped callable - would ignore behavior added by the wrapper. - """ - loader = getattr(param, "weight_loader", None) - if loader is None or isinstance(loader, partial): - return None - - owner = loader.__self__ if inspect.ismethod(loader) else None - loader_fn = loader.__func__ if inspect.ismethod(loader) else loader - - # Lazy imports avoid making the weight-transfer registry import all model - # executor layers during normal vLLM startup. - from vllm.model_executor.layers.linear import ( - ColumnParallelLinear, - ReplicatedLinear, - RowParallelLinear, - ) - from vllm.model_executor.model_loader.weight_utils import default_weight_loader - - replicated_loaders = { - default_weight_loader, - ReplicatedLinear.weight_loader, - } - column_loaders = { - ColumnParallelLinear.weight_loader, - ColumnParallelLinear.weight_loader_v2, - } - row_loaders = { - RowParallelLinear.weight_loader, - RowParallelLinear.weight_loader_v2, - } - - # These attributes signal packing, a transform, or pre-sharded checkpoint - # storage. Such semantics cannot be reproduced by a plain M2N placement. - # Check them before accepting replicated loaders too: model-level loading - # may transform a tensor before handing it to an otherwise plain copier. - metadata_attrs = ("packed_dim", "packed_factor", "marlin_tile_size") - flag_attrs = ("needs_scalar_to_array", "is_sharded_weight", "is_transposed") - if any(getattr(param, attr, None) is not None for attr in metadata_attrs): - return None - if any(bool(getattr(param, attr, False)) for attr in flag_attrs): - return None - - if loader_fn in replicated_loaders: - return REPLICATE - if loader_fn not in column_loaders | row_loaders: - return None - - if owner is None or getattr(owner, "tp_size", None) != shard_axis_size: - return None - - attr = "output_dim" if loader_fn in column_loaders else "input_dim" - dim = getattr(param, attr, None) - return dim if isinstance(dim, int) and dim >= 0 else None - - -def _validated_shard_dim( - global_shape: Sequence[int], - local_shape: Sequence[int], - declared_dim: int, - shard_axis_size: int, -) -> int | None: - """Validate a loader-declared shard dimension against both shapes.""" - if len(global_shape) != len(local_shape): - return None - - if declared_dim == REPLICATE: - return REPLICATE if tuple(global_shape) == tuple(local_shape) else None - if declared_dim >= len(global_shape): - return None - for dim, (whole, local) in enumerate(zip(global_shape, local_shape)): - expected = local * shard_axis_size if dim == declared_dim else local - if whole != expected: - return None - return declared_dim - - -def resolve_parameter_destinations( - model: torch.nn.Module, - names: Sequence[str], - dtypes: Sequence[torch.dtype], - shapes: Sequence[Sequence[int]], - *, - num_workers: int, - shard_axis_size: int, - allow_direct: bool, -) -> list[M2NDestination]: - """Build this worker's destination plan, one entry per checkpoint parameter. - - Placements are relative to the inference mesh: axis 0 replicates and axis 1 - shards. `allow_direct=False` forces every parameter onto the fallback path; - the engine sets it for pipeline-parallel or quantized deployments, where a - parameter's local shape is not simply the checkpoint shape split across the - shard axis. Parameters below a model type with known model-level checkpoint - preprocessing also fall back, since bypassing `load_weights` would skip the - transformation. - """ - params = dict(model.named_parameters()) if allow_direct else {} - blocked_param_ids = { - id(param) - for module in model.modules() - if _module_type_name(module) in _DIRECT_WEIGHT_LOADING_BLOCKLIST - for param in module.parameters() - } - - destinations: list[M2NDestination] = [] - for name, dtype, shape in zip(names, dtypes, shapes): - param = params.get(name) - dim = None - if ( - param is not None - and id(param) not in blocked_param_ids - and param.dtype == dtype - and param.data.is_contiguous() - and num_workers % shard_axis_size == 0 - ): - declared_dim = _base_loader_shard_dim(param, shard_axis_size) - if declared_dim is not None: - dim = _validated_shard_dim( - shape, param.shape, declared_dim, shard_axis_size - ) - - if dim is None: - # A replicated parameter is identical on every rank, so it needs no - # placement of its own — REPLICATED lets resolve_layout spread it - # over all the workers regardless of how the mesh is factored. - destinations.append(M2NDestination(name, REPLICATED, None)) - elif dim == REPLICATE: - assert param is not None - destinations.append(M2NDestination(name, REPLICATED, param.data)) - else: - assert param is not None - destinations.append( - M2NDestination(name, _destination_placements(dim), param.data) - ) - - # Layerwise reload finalizes a module as a unit. Mixing direct and fallback - # parameters within one module can copy stale storage over the direct load. - fallback_modules = { - name.rpartition(".")[0] - for name, destination in zip(names, destinations) - if not destination.direct and name in params - } - destinations = [ - M2NDestination(destination.name, REPLICATED, None) - if destination.direct - and destination.name.rpartition(".")[0] in fallback_modules - else destination - for destination in destinations - ] - - num_direct = sum(d.direct for d in destinations) - parameter_bytes = [ - math.prod(shape) * dtype.itemsize for dtype, shape in zip(dtypes, shapes) - ] - total_bytes = sum(parameter_bytes) - direct_bytes = sum( - nbytes - for destination, nbytes in zip(destinations, parameter_bytes) - if destination.direct - ) - direct_byte_percentage = 100 * direct_bytes / total_bytes if total_bytes else 0 - logger.info( - "nccl_m2n destination plan: %d/%d parameters resharded directly into " - "the model, %d via full-tensor fallback; direct byte coverage: " - "%d/%d bytes (%.1f%%)", - num_direct, - len(destinations), - len(destinations) - num_direct, - direct_bytes, - total_bytes, - direct_byte_percentage, - ) - return destinations diff --git a/vllm/distributed/weight_transfer/m2n_source.py b/vllm/distributed/weight_transfer/m2n_source.py deleted file mode 100644 index d9c9e9194a..0000000000 --- a/vllm/distributed/weight_transfer/m2n_source.py +++ /dev/null @@ -1,207 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Trainer-side weight sources for the NCCL M2N backend. - -m2n plans a transfer from both sides' layouts, so the trainer has to say how it -holds each parameter — which the base `WeightSource` / `ParamMeta` pair does not -express. This module supplies the m2n flavor of both, and the DTensor-backed -implementation that covers the common trainer. Each parameter owns its source -mesh and placements because dense and expert weights can use different -factorizations over the same trainer ranks. -""" - -from collections.abc import Callable, Iterator, Sequence -from dataclasses import dataclass -from typing import Any - -import torch - -from vllm.distributed.weight_transfer.base import WeightSource -from vllm.distributed.weight_transfer.m2n_common import ( - REPLICATE, - REPLICATED, - M2NLayout, - M2NMesh, - M2NParamMeta, - Placements, -) - -__all__ = [ - "M2NWeightSource", - "M2NManifestEntry", - "ManifestM2NWeightSource", - "DTensorModuleSource", - "mesh_from_tensor", - "placements_from_tensor", -] - - -class M2NWeightSource(WeightSource): - """A `WeightSource` that also describes how the trainer holds its weights. - - Unlike `ModuleSource`, iteration yields each rank's **local shard**, not a - materialized full tensor: gathering is exactly the cost m2n removes. - """ - - def metadata(self) -> list[M2NParamMeta]: # type: ignore[override] - """Name, dtype, full shape, and source layout for each parameter.""" - raise NotImplementedError - - def __iter__(self) -> Iterator[tuple[str, Any]]: # type: ignore[override] - """Yield `(name, local buffer)` pairs in metadata order. - - Providers must enqueue materialization on the caller's current CUDA - stream, or return a buffer whose writes are already visible to it. - Buffers may be torch tensors, CUDA Array Interface objects, or raw - device pointers accepted by nccl4py. Providers own correct - communicator-device placement. Raw-pointer providers additionally own - allocation lifetime, dtype, and shape correctness. - """ - raise NotImplementedError - - -@dataclass(frozen=True) -class M2NManifestEntry: - """One immutable source descriptor with a lazy rank-local tensor view.""" - - key: str - dtype: torch.dtype - global_shape: tuple[int, ...] - source_layout: M2NLayout - local_tensor: Callable[[], Any] - - def __post_init__(self) -> None: - object.__setattr__(self, "global_shape", tuple(self.global_shape)) - if not self.key: - raise ValueError("M2N manifest key must not be empty") - if not isinstance(self.dtype, torch.dtype): - raise TypeError( - f"manifest entry '{self.key}' has invalid dtype {self.dtype!r}" - ) - if not isinstance(self.source_layout, M2NLayout): - raise TypeError( - f"manifest entry '{self.key}' source_layout must be an M2NLayout" - ) - if not callable(self.local_tensor): - raise TypeError( - f"manifest entry '{self.key}' local_tensor must be callable" - ) - - @property - def metadata(self) -> M2NParamMeta: - """Return the serializable portion consumed during initialization.""" - return M2NParamMeta( - self.key, - self.dtype, - self.global_shape, - self.source_layout, - ) - - -class ManifestM2NWeightSource(M2NWeightSource): - """An ordered M2N source backed by lazy, rank-local manifest entries.""" - - def __init__(self, entries: Sequence[M2NManifestEntry]) -> None: - self._entries = tuple(entries) - self._metadata = tuple(entry.metadata for entry in self._entries) - - def metadata(self) -> list[M2NParamMeta]: # type: ignore[override] - """Return a fresh list while preserving the immutable manifest order.""" - return list(self._metadata) - - def __iter__(self) -> Iterator[tuple[str, Any]]: # type: ignore[override] - """Materialize one rank-local buffer at a time.""" - for entry in self._entries: - yield entry.key, entry.local_tensor() - - -def _placement_code(placement: Any) -> int: - """Map a `torch.distributed` placement onto an m2n placement code.""" - name = type(placement).__name__ - if name == "Replicate": - return REPLICATE - if name == "Shard": - return int(placement.dim) - raise ValueError( - f"nccl_m2n cannot express the {name} placement; only Replicate and " - "Shard are supported" - ) - - -def mesh_from_tensor(tensor: torch.Tensor, num_trainer_ranks: int) -> M2NMesh: - """The mesh a parameter lives on, as an `M2NMesh`. - - Tensors without DeviceMesh metadata are treated as replicated. Implicitly - sharded tensors require a custom source with an explicit layout. - """ - device_mesh = getattr(tensor, "device_mesh", None) - if device_mesh is None: - return M2NMesh((num_trainer_ranks, 1), 0) - - grid = device_mesh.mesh - ranks = grid.flatten().tolist() - if ranks != list(range(len(ranks))): - raise ValueError( - "nccl_m2n requires the trainer's device mesh to cover the " - f"contiguous rank interval [0, {len(ranks)}); got {ranks}" - ) - if grid.ndim > 2: - raise ValueError( - f"nccl_m2n supports 1-D and 2-D device meshes, got {grid.ndim}-D" - ) - dims = tuple(grid.shape) - return M2NMesh(dims if len(dims) == 2 else (dims[0], 1), 0) - - -def placements_from_tensor(tensor: torch.Tensor) -> Placements | None: - """How a parameter is placed over its mesh, or `REPLICATED`. - - `REPLICATED` is returned rather than a `(REPLICATE, REPLICATE)` pair: m2n - cannot express that, and the size-1-axis encoding it needs instead is - applied later by `resolve_layout`, which owns that workaround. - """ - placements = getattr(tensor, "placements", None) - if placements is None: - return REPLICATED - codes = [_placement_code(p) for p in placements] - if all(code == REPLICATE for code in codes): - return REPLICATED - if len(codes) == 1: - return (codes[0], REPLICATE) - return (codes[0], codes[1]) - - -class DTensorModuleSource(M2NWeightSource): - """`M2NWeightSource` over `module.named_parameters()`. - - Covers both the FSDP/DTensor trainer (placement read off each parameter, - local shard yielded via `to_local()`) and the plain replicated trainer, with - no special casing. Trainers with a custom producer (a Megatron export, MoE - re-fusing) subclass `M2NWeightSource` instead. - """ - - def __init__(self, module: torch.nn.Module, num_trainer_ranks: int) -> None: - """Wrap `module`; `num_trainer_ranks` sizes the mesh for plain tensors.""" - self._module = module - self._num_trainer_ranks = num_trainer_ranks - - def metadata(self) -> list[M2NParamMeta]: # type: ignore[override] - """Read each global shape and source layout without gathering shards.""" - return [ - M2NParamMeta( - name, - p.dtype, - tuple(p.shape), - M2NLayout( - mesh_from_tensor(p, self._num_trainer_ranks), - placements_from_tensor(p), - ), - ) - for name, p in self._module.named_parameters() - ] - - def __iter__(self) -> Iterator[tuple[str, torch.Tensor]]: - """Yield each parameter's local shard (`to_local()`), or the tensor itself.""" - for name, param in self._module.named_parameters(): - to_local = getattr(param, "to_local", None) - yield name, (to_local() if callable(to_local) else param) diff --git a/vllm/distributed/weight_transfer/m2n_trainer.py b/vllm/distributed/weight_transfer/m2n_trainer.py deleted file mode 100644 index 19dd973c73..0000000000 --- a/vllm/distributed/weight_transfer/m2n_trainer.py +++ /dev/null @@ -1,628 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Trainer-side weight transfer engine for the NCCL M2N backend. - -Symmetric to `M2NWeightTransferEngine` but in the training process. Every -trainer rank joins the shared communicator and runs every reshard, sending its -own local shard; only rank 0 touches the inference control plane. - -Workers join the communicator during init and run reshard from inside -`update_weights`, so those two RPCs overlap with work on this side. Both run on -a helper thread and are joined afterwards, which keeps that overlap inside the -engine -- where the `TrainerWeightTransferEngine` contract puts it -- rather than -in every caller. -""" - -import socket -from collections.abc import Mapping -from concurrent.futures import Future, ThreadPoolExecutor -from dataclasses import dataclass, field -from typing import TYPE_CHECKING, Any, ClassVar, Self, cast - -import torch - -from vllm.distributed.weight_transfer.base import ( - TrainerInitInfo, - TrainerWeightTransferEngine, - VLLMWeightSyncClient, - WeightSource, -) -from vllm.distributed.weight_transfer.m2n_common import ( - M2N_WIRE_SCHEMA_VERSION, - M2NMesh, - M2NParamMeta, - M2NWireParam, - Placements, - check_complete_plan_agreement, - check_data_plane_agreement, - check_runtime_ready_agreement, - check_source_plan_agreement, - check_transferable, - create_m2n_communicator, - expected_local_shape, - import_m2n, - prepare_m2n_local_runtime, - publish_destination_placements, - resolve_layout, - source_plan_digest, - to_mesh, - to_placements, - validate_local_buffer, - validate_transfer_plan, -) -from vllm.distributed.weight_transfer.m2n_source import M2NWeightSource -from vllm.distributed.weight_transfer.nccl_common import ( - decode_nccl_unique_id, - stateless_init_metadata_group, -) -from vllm.logger import init_logger - -if TYPE_CHECKING: - from vllm.distributed.utils import StatelessProcessGroup - -logger = init_logger(__name__) - -__all__ = ["M2NTrainerInitInfo", "M2NTrainerWeightTransferEngine"] - - -@dataclass(kw_only=True) -class M2NTrainerInitInfo(TrainerInitInfo): - """Trainer-side init info for nccl_m2n. - - `rank` (from `TrainerInitInfo`) is this trainer process's rank; it is also - its rank in the shared communicator, since the trainer occupies - `[0, num_trainer_ranks)`. Rank 0 drives the control plane. - """ - - backend: ClassVar[str] = "nccl_m2n" - - master_address: str - master_port: int - world_size: int - """Trainer ranks + all inference workers.""" - num_trainer_ranks: int = 1 - dst_mesh_dims: tuple[int, int] | None = None - """How the inference ranks are laid out: axis 0 replicates and axis 1 - shards. The trainer declares it so both sides describe the destination - identically; defaults to a flat `(num_workers, 1)`, which is all a - replicated destination needs. - - Destination meshes remain common to every parameter; per-parameter - destination meshes are intentionally future work. - """ - nccl_unique_id_b64: str | None = field(default=None, repr=False) - """Optional UID for the NCCL data plane; TCP remains the metadata plane.""" - max_cta: int | None = None - listen_socket: socket.socket | None = field(default=None, repr=False, compare=False) - """Optional rank-zero TCPStore reservation; never sent to workers.""" - - @property - def destination_mesh_dims(self) -> tuple[int, int]: - """`dst_mesh_dims`, or a flat mesh over every inference worker.""" - if self.dst_mesh_dims is not None: - return cast(tuple[int, int], tuple(self.dst_mesh_dims)) - return (self.world_size - self.num_trainer_ranks, 1) - - def __post_init__(self) -> None: - """Reject rank counts or destination meshes that cannot form a group.""" - if not isinstance(self.master_address, str) or not self.master_address: - raise ValueError("`master_address` must name the metadata rendezvous") - if ( - not isinstance(self.master_port, int) - or isinstance(self.master_port, bool) - or not 0 < self.master_port < 65536 - ): - raise ValueError("`master_port` must be a valid metadata port") - if not 0 < self.num_trainer_ranks < self.world_size: - raise ValueError( - f"`num_trainer_ranks` ({self.num_trainer_ranks}) must leave at " - f"least one worker in world_size {self.world_size}" - ) - num_workers = self.world_size - self.num_trainer_ranks - if self.dst_mesh_dims is not None: - dims = tuple(self.dst_mesh_dims) - try: - mesh = M2NMesh(cast(tuple[int, int], dims), self.num_trainer_ranks) - except (TypeError, ValueError) as exc: - raise ValueError(f"invalid `dst_mesh_dims` {dims}: {exc}") from exc - if mesh.size != num_workers: - raise ValueError( - f"`dst_mesh_dims` {dims} must be two dims covering the " - f"{num_workers} inference workers" - ) - if not 0 <= self.rank < self.num_trainer_ranks: - raise ValueError( - f"trainer `rank` ({self.rank}) must be below " - f"`num_trainer_ranks` ({self.num_trainer_ranks})" - ) - if self.listen_socket is not None: - if self.rank != 0: - raise ValueError("only trainer rank zero may reserve the TCPStore") - if not isinstance(self.listen_socket, socket.socket): - raise TypeError("listen_socket must be a socket.socket") - if self.listen_socket.fileno() < 0: - raise ValueError("listen_socket is already closed") - - @property - def nccl_unique_id_bytes(self) -> bytes | None: - if self.nccl_unique_id_b64 is None: - return None - return decode_nccl_unique_id( - master_address=None, - master_port=None, - nccl_unique_id_b64=self.nccl_unique_id_b64, - ctx="M2NTrainerInitInfo data plane", - ) - - -class M2NTrainerWeightTransferEngine(TrainerWeightTransferEngine[M2NTrainerInitInfo]): - """Trainer-side engine: sends every rank's local shard, once per parameter. - - Called on every trainer rank. All ranks join the communicator and run every - reshard; only rank 0 touches the control plane. - """ - - init_info_cls = M2NTrainerInitInfo - - def __init__( - self, - *, - client: VLLMWeightSyncClient, - source: M2NWeightSource, - is_sender: bool, - ) -> None: - """Hold the client and source; `trainer_init` fills in the group.""" - super().__init__(client=client, source=source, is_sender=is_sender) - self.group: Any = None - self._m2n: Any = None - self._handle: Any = None - self._metas: list[M2NParamMeta] = [] - self._wire_params: list[M2NWireParam] = [] - self._source_digest = "" - self._dst_mesh: M2NMesh | None = None - self._dst_placements: list[Placements | None] = [] - self._metadata_group: StatelessProcessGroup | None = None - self._num_trainer_ranks = 0 - - @classmethod - def trainer_init( - cls, - init_info: M2NTrainerInitInfo, - *, - client: VLLMWeightSyncClient, - source: WeightSource | None = None, - ) -> Self: - """Build the engine and rendezvous with the inference side. - - Runs on *every* trainer rank. Rank 0 additionally ships the transfer - plan to the workers and drives the control plane; the other ranks only - build local state and join the communicator. Every rank participates - in every reshard and sends its local shard. - - Build a local validation envelope, start the worker RPC without - waiting, join the metadata rendezvous, and require unanimous source, - data-plane, and runtime readiness before constructing NCCL. - """ - engine = cls( - client=client, - source=cast(M2NWeightSource, source), - is_sender=init_info.is_sender, - ) - engine._initialize(init_info, source) - return engine - - def _initialize( - self, - init_info: M2NTrainerInitInfo, - source: WeightSource | None, - ) -> None: - """Run source validation, metadata agreement, and NCCL rendezvous.""" - self._num_trainer_ranks = init_info.num_trainer_ranks - local_source_error: str | None = None - try: - if not isinstance(source, M2NWeightSource): - raise TypeError( - "nccl_m2n needs per-parameter layouts, so its source must " - f"be an M2NWeightSource; got {type(source).__name__}" - ) - self._m2n = import_m2n() - self._prepare_source_plan(source, init_info.num_trainer_ranks) - except Exception as exc: - local_source_error = f"{type(exc).__name__}: {exc}" - self._metas = [] - self._wire_params = [] - self._source_digest = source_plan_digest([]) - - executor: ThreadPoolExecutor | None = None - init_future: Future[None] | None = None - if self.is_sender: - executor = ThreadPoolExecutor( - max_workers=1, thread_name_prefix="m2n-weight-sync" - ) - init_future = executor.submit( - self.client.init_weight_transfer_engine, - self._worker_init_info(init_info), - ) - handle = None - group = None - try: - reservation = init_info.listen_socket - try: - metadata_group = stateless_init_metadata_group( - init_info.master_address, - init_info.master_port, - init_info.rank, - init_info.world_size, - listen_socket=reservation, - ) - finally: - init_info.listen_socket = None - if reservation is not None: - reservation.close() - plan_digest = check_source_plan_agreement( - metadata_group, - self._wire_params, - expected_digest=self._source_digest, - local_error=local_source_error, - ) - - # Retain the common destination mesh and the merged NCCL publication - # of per-parameter placements. - self._dst_mesh = M2NMesh( - init_info.destination_mesh_dims, init_info.num_trainer_ranks - ) - unique_id_bytes: bytes | None = None - data_plane_error: str | None = None - try: - unique_id_bytes = init_info.nccl_unique_id_bytes - except Exception as exc: - data_plane_error = f"{type(exc).__name__}: {exc}" - check_data_plane_agreement( - metadata_group, - unique_id_bytes, - data_plane_error, - ) - - runtime_error: str | None = None - nccl = None - device = None - try: - nccl, device, handle = prepare_m2n_local_runtime( - self._m2n, init_info.max_cta - ) - except Exception as exc: - runtime_error = f"{type(exc).__name__}: {exc}" - check_runtime_ready_agreement(metadata_group, runtime_error) - assert nccl is not None and device is not None and handle is not None - group = create_m2n_communicator(nccl, metadata_group, unique_id_bytes) - destination_placements = self._receive_destination_plan( - group, - init_info.num_trainer_ranks, - torch.device("cuda", device), - ) - - complete_plan_error: str | None = None - try: - assert self._dst_mesh is not None - validate_transfer_plan( - self._metas, self._dst_mesh, destination_placements - ) - except Exception as exc: - complete_plan_error = f"{type(exc).__name__}: {exc}" - assert self._dst_mesh is not None - check_complete_plan_agreement( - metadata_group, - plan_digest, - self._dst_mesh, - destination_placements, - complete_plan_error, - ) - - if init_future is not None: - init_future.result() - - logger.info( - "nccl_m2n trainer ready: %d parameters, trainer ranks [0, %d), " - "inference ranks [%d, %d)", - len(self._metas), - init_info.num_trainer_ranks, - init_info.num_trainer_ranks, - init_info.world_size, - ) - logger.info("nccl_m2n source plan digest: %s", plan_digest) - except BaseException: - if group is not None: - try: - # Transport cleanup only; reload validity belongs to the - # model-update lifecycle, not this backend. - group.abort() - except BaseException: - logger.exception( - "failed to abort partially initialized nccl_m2n communicator" - ) - if handle is not None: - try: - handle.destroy() - except BaseException: - logger.exception( - "failed to destroy partially initialized nccl_m2n handle" - ) - raise - finally: - if executor is not None: - executor.shutdown(wait=False, cancel_futures=True) - - self._metadata_group = metadata_group - self._handle = handle - self.group = group - self._dst_placements = destination_placements - - def _receive_destination_plan( - self, group: Any, first_worker_rank: int, device: torch.device - ) -> list[Placements | None]: - """Receive the inference-side per-parameter placements.""" - # The first inference worker publishes one placement per parameter. - # Every trainer rank receives the complete plan in this single, - # initialization-time broadcast. - placements = publish_destination_placements( - group, - device, - torch.cuda.current_stream(), - first_worker_rank, - None, - len(self._metas), - ) - return placements - - def _worker_init_info(self, init_info: M2NTrainerInitInfo) -> dict[str, Any]: - """Handshake payload: rendezvous, both meshes, and the transfer plan.""" - payload: dict[str, Any] = { - "schema_version": M2N_WIRE_SCHEMA_VERSION, - "master_address": init_info.master_address, - "master_port": init_info.master_port, - "nccl_unique_id_b64": init_info.nccl_unique_id_b64, - "rank_offset": init_info.num_trainer_ranks, - "world_size": init_info.world_size, - "dst_mesh_dims": list(init_info.destination_mesh_dims), - "source_digest": self._source_digest, - "params": [param.to_dict() for param in self._wire_params], - "max_cta": init_info.max_cta, - } - return {key: value for key, value in payload.items() if value is not None} - - def _prepare_source_plan( - self, source: M2NWeightSource, num_trainer_ranks: int - ) -> None: - """Build and validate this rank's immutable source plan.""" - self._metas = source.metadata() - names: set[str] = set() - for meta in self._metas: - if not isinstance(meta, M2NParamMeta): - raise TypeError( - "nccl_m2n source metadata must contain M2NParamMeta " - f"entries, got {type(meta).__name__}" - ) - if meta.name in names: - raise ValueError( - "nccl_m2n source metadata contains duplicate parameter " - f"'{meta.name}'" - ) - names.add(meta.name) - check_transferable(meta.name, meta.dtype, meta.shape) - layout = meta.source_layout - if layout.mesh.start_rank != 0: - raise ValueError( - f"parameter '{meta.name}' source mesh must start at rank 0, " - f"got {layout.mesh.start_rank}" - ) - if layout.mesh.size != num_trainer_ranks: - raise ValueError( - f"parameter '{meta.name}' source mesh covers " - f"{layout.mesh.size} ranks, but there are " - f"{num_trainer_ranks} trainer ranks" - ) - expected_local_shape(meta) - self._wire_params = [ - M2NWireParam( - name=meta.name, - dtype_name=str(meta.dtype).split(".")[-1], - shape=meta.shape, - src_mesh_dims=meta.source_layout.mesh.dims, - src_placements=meta.source_layout.placements, - ) - for meta in self._metas - ] - self._source_digest = source_plan_digest(self._wire_params) - - def send_weights(self) -> None: - """Drive one update round: start, reshard concurrently, then finish.""" - if self._handle is None or self.group is None: - raise RuntimeError("nccl_m2n trainer engine is not initialized") - - if not self.is_sender: - try: - self._send() - except BaseException: - try: - self._receive_round_outcome() - except BaseException: - logger.exception( - "failed to consume nccl_m2n round outcome after local failure" - ) - raise - self._receive_round_outcome() - return - - try: - self.client.start_weight_update() - executor = ThreadPoolExecutor( - max_workers=1, thread_name_prefix="m2n-weight-sync" - ) - try: - update = executor.submit( - self.client.update_weights, - {"names": [m.name for m in self._metas]}, - ) - # Surface an immediate worker-side rejection before trainers - # enter the collective sequence. - if update.done(): - update.result() - self._send() - update.result() - finally: - executor.shutdown(wait=False, cancel_futures=True) - self.client.finish_weight_update() - except BaseException as exc: - try: - self._publish_round_outcome(exc) - except BaseException: - logger.exception("failed to publish nccl_m2n round failure") - raise - - self._publish_round_outcome(None) - - def _publish_round_outcome(self, failure: BaseException | None) -> None: - """Tell non-sender trainer ranks whether worker finalization passed.""" - if self._num_trainer_ranks <= 1: - return - if self._metadata_group is None: - raise RuntimeError("nccl_m2n metadata group is unavailable") - envelope = { - "ok": failure is None, - "error": ( - None if failure is None else f"{type(failure).__name__}: {failure}" - ), - } - for rank in range(1, self._num_trainer_ranks): - self._metadata_group.send_obj(envelope, rank) - # Keep rank zero's TCPStore alive until every trainer consumed its - # outcome. This also keeps the per-rank message counters aligned. - expected_ack = {"phase": "round_outcome_consumed"} - for rank in range(1, self._num_trainer_ranks): - ack = self._metadata_group.recv_obj(rank) - if ack != expected_ack: - raise RuntimeError( - f"trainer rank {rank} sent an invalid round-outcome acknowledgement" - ) - - def _receive_round_outcome(self) -> None: - """Wait until trainer rank zero observed every worker's finish RPC.""" - if self._metadata_group is None: - raise RuntimeError("nccl_m2n metadata group is unavailable") - outcome = self._metadata_group.recv_obj(0) - self._metadata_group.send_obj({"phase": "round_outcome_consumed"}, 0) - if not isinstance(outcome, Mapping) or set(outcome) != {"ok", "error"}: - raise RuntimeError("trainer rank zero sent an invalid round outcome") - ok = outcome["ok"] - error = outcome["error"] - if not isinstance(ok, bool) or ok != (error is None): - raise RuntimeError("trainer rank zero sent an invalid round outcome") - if not ok: - if not isinstance(error, str) or not error: - raise RuntimeError("trainer rank zero sent an invalid round error") - raise RuntimeError( - "nccl_m2n worker update failed on trainer rank zero: " + error - ) - - def _send(self) -> None: - """Reshard each local shard into its worker-planned destination.""" - assert self.group is not None - assert self._dst_mesh is not None - assert self.source is not None - m2n = self._m2n - stream = torch.cuda.current_stream() - - device_index = torch.accelerator.current_device_index() - # Optimizer work may have used another CUDA stream. Make prior writes - # visible before a provider materializes current-stream local shards. - torch.accelerator.synchronize(device_index) - inflight_buffers: list[Any] = [] - sent = 0 - for index, (name, buffer) in enumerate(self.source): - if index >= len(self._metas): - raise RuntimeError( - "weight source yielded more parameters than metadata() " - f"declared (first extra parameter: '{name}')" - ) - meta = self._metas[index] - if name != meta.name: - raise RuntimeError( - f"weight source yielded '{name}' at position {index}, but " - f"its metadata declared '{meta.name}'; the source must " - "iterate in a stable order" - ) - validate_local_buffer(m2n, meta, buffer) - src_mesh, src_placements = resolve_layout( - meta.source_layout.mesh, meta.source_layout.placements - ) - dst_mesh, dst_placements = resolve_layout( - self._dst_mesh, self._dst_placements[index] - ) - raw_pointer_args = {} - if isinstance(buffer, int): - raw_pointer_args = { - "src_local_shape": expected_local_shape(meta), - "src_dtype": meta.dtype, - } - m2n.reshard( - buffer, - None, - self.group, - stream, - src_mesh=to_mesh(m2n, src_mesh), - src_placements=to_placements(m2n, src_placements), - dst_mesh=to_mesh(m2n, dst_mesh), - dst_placements=to_placements(m2n, dst_placements), - handle=self._handle, - **raw_pointer_args, - ) - # Let allocator-aware buffers recycle storage in stream order. - # Other object owners must stay alive until the final synchronize; - # raw-pointer lifetime remains the source's responsibility. - record_stream = getattr(buffer, "record_stream", None) - if callable(record_stream): - record_stream(stream) - elif not isinstance(buffer, int): - inflight_buffers.append(buffer) - sent += 1 - - if sent != len(self._metas): - missing = self._metas[sent].name - raise RuntimeError( - f"weight source yielded {sent} parameters, but metadata() " - f"declared {len(self._metas)} (first missing: '{missing}')" - ) - - # As in the other weight-transfer backends, wait before returning - # rather than relying on same-stream ordering. The caller may then - # safely mutate parameters or continue work on another stream. - stream.synchronize() - - def shutdown(self) -> None: - """Finish pending GPU work and release M2N communication state.""" - handle, self._handle = self._handle, None - group, self.group = self.group, None - self._metadata_group = None - self._dst_placements = [] - self._dst_mesh = None - self._m2n = None - - first_error: BaseException | None = None - - def attempt(action: Any, message: str) -> None: - nonlocal first_error - try: - action() - except BaseException as exc: - if first_error is None: - first_error = exc - logger.exception(message) - - if handle is not None or group is not None: - attempt(torch.accelerator.synchronize, "failed to synchronize nccl_m2n") - if handle is not None: - attempt(handle.destroy, "failed to destroy nccl_m2n handle") - if group is not None: - attempt(group.abort, "failed to abort nccl_m2n communicator") - if first_error is not None: - raise first_error diff --git a/vllm/distributed/weight_transfer/modelexpress_engine.py b/vllm/distributed/weight_transfer/modelexpress_engine.py deleted file mode 100644 index 1172542e85..0000000000 --- a/vllm/distributed/weight_transfer/modelexpress_engine.py +++ /dev/null @@ -1,41 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -"""Re-export the weight-transfer backend maintained by ModelExpress. - -Install ModelExpress from the main branch's modelexpress_client/python -subdirectory. -""" - -try: - from modelexpress import configure_vllm_logging - from modelexpress_rl.inference.engines.vllm.weight_transfer_engine import ( - ModelExpressWeightTransferEngine, - ModelExpressWeightTransferInitInfo, - ModelExpressWeightTransferUpdateInfo, - ) -except ModuleNotFoundError as exc: - if exc.name not in { - "modelexpress", - "modelexpress_rl", - "modelexpress_rl.inference", - "modelexpress_rl.inference.engines", - "modelexpress_rl.inference.engines.vllm", - "modelexpress_rl.inference.engines.vllm.weight_transfer_engine", - }: - raise - raise ImportError( - "The 'modelexpress' weight transfer backend requires ModelExpress. " - "Install it with `uv pip install " - "'git+https://github.com/ai-dynamo/modelexpress@main" - "#subdirectory=modelexpress_client/python'`." - ) from exc - -configure_vllm_logging() - -__all__ = [ - "ModelExpressWeightTransferEngine", - "ModelExpressWeightTransferInitInfo", - "ModelExpressWeightTransferUpdateInfo", -] diff --git a/vllm/distributed/weight_transfer/nccl_common.py b/vllm/distributed/weight_transfer/nccl_common.py deleted file mode 100644 index b0ac285cf1..0000000000 --- a/vllm/distributed/weight_transfer/nccl_common.py +++ /dev/null @@ -1,299 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Shared NCCL initialization helpers for weight transfer engines. - -The dense (`NCCLWeightTransferEngine`) and sparse -(`SparseNCCLWeightTransferEngine`) backends are independent engines that share -*only* their process-group initialization. That common logic lives here so the -sparse engine does not have to subclass the dense one. -""" - -import socket -from dataclasses import asdict, dataclass, field -from typing import TYPE_CHECKING, Protocol - -import pybase64 as base64 -import torch - -if TYPE_CHECKING: - from vllm.config.parallel import ParallelConfig - from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator - from vllm.distributed.utils import StatelessProcessGroup - -from vllm.distributed.device_communicators.pynccl_wrapper import ( - NCCL_UNIQUE_ID_BYTES, -) -from vllm.distributed.weight_transfer.base import WeightTransferInitInfo -from vllm.distributed.weight_transfer.packed_tensor import ( - DEFAULT_PACKED_BUFFER_SIZE_BYTES, - DEFAULT_PACKED_NUM_BUFFERS, -) -from vllm.utils.nccl import unpinned_nccl_env - - -def decode_nccl_unique_id( - *, - master_address: str | None, - master_port: int | None, - nccl_unique_id_b64: str | None, - ctx: str, -) -> bytes | None: - """Validate the rendezvous mode and decode a pre-shared unique id.""" - has_uid = bool(nccl_unique_id_b64) - has_addr = master_address is not None - has_port = master_port is not None - - if has_uid and (has_addr or has_port): - raise ValueError( - f"{ctx}: pass nccl_unique_id_b64 OR master_address/master_port, not both" - ) - if not has_uid: - if not (has_addr and has_port): - raise ValueError( - f"{ctx}: need nccl_unique_id_b64, or both master_address and " - f"master_port (got master_address={master_address!r}, " - f"master_port={master_port!r})" - ) - return None - try: - decoded = base64.b64decode(nccl_unique_id_b64, validate=True) - except (ValueError, TypeError) as e: - raise ValueError( - f"{ctx}: nccl_unique_id_b64 is not valid standard base64 " - f"(URL-safe base64 is not accepted)" - ) from e - if len(decoded) != NCCL_UNIQUE_ID_BYTES: - raise ValueError( - f"{ctx}: nccl_unique_id_b64 must decode to {NCCL_UNIQUE_ID_BYTES} " - f"bytes, got {len(decoded)}" - ) - return decoded - - -@dataclass(kw_only=True) -class NCCLWeightTransferInitInfo(WeightTransferInitInfo): - """Worker-side initialization info for NCCL-based weight transfer backends. - - Keyword-only (`kw_only`): adding the optional `nccl_unique_id_b64` field - means the rendezvous fields can no longer keep a fixed positional slot, so a - stale positional call fails loudly instead of silently swapping arguments. - - Provide exactly one rendezvous mode: - - * ``master_address`` + ``master_port`` -- TCPStore / ``StatelessProcessGroup`` - rendezvous (requires torch on every rank, including the trainer), or - * ``nccl_unique_id_b64`` -- standard (RFC 4648, *not* URL-safe) base64 of the - 128 raw bytes from ``ncclGetUniqueId``, for torch-free trainers (e.g. JAX) - that mint the unique id out of band and share it (over HTTP, etc.). Note a - JAX peer must use ``base64.b64encode``, not ``urlsafe_b64encode``. - - On the unique-id path all ranks must enter init concurrently (there is no - store barrier), and every peer must honor the warm-up handshake: the - worker's communicator issues a one-element ``all_reduce`` immediately after - ``ncclCommInitRank`` (see ``PyNcclCommunicator.from_unique_id_bytes``), so a - foreign peer must issue a matching one-element ``all_reduce`` before any - other collective or all ranks deadlock. - """ - - master_address: str | None = None - master_port: int | None = None - rank_offset: int - world_size: int - nccl_unique_id_b64: str | None = field(default=None, repr=False) - packed: bool = False - packed_buffer_size_bytes: int = DEFAULT_PACKED_BUFFER_SIZE_BYTES - packed_num_buffers: int = DEFAULT_PACKED_NUM_BUFFERS - - def __post_init__(self) -> None: - _ = self.nccl_unique_id_bytes - - @property - def nccl_unique_id_bytes(self) -> bytes | None: - return decode_nccl_unique_id( - master_address=self.master_address, - master_port=self.master_port, - nccl_unique_id_b64=self.nccl_unique_id_b64, - ctx="NCCLWeightTransferInitInfo", - ) - - -def worker_init_payload(init_info: NCCLWeightTransferInitInfo) -> dict: - """Serialize a worker init info for `init_weight_transfer_engine`, dropping - the unset rendezvous field (the UID in TCP mode) so the wire payload carries - only the mode actually in use. Shared by the dense and sparse trainer - engines so the two cannot drift.""" - return {key: value for key, value in asdict(init_info).items() if value is not None} - - -class NCCLRendezvous(Protocol): - """The TCP rendezvous fields `trainer_init` needs. - - Structural so each backend can keep its own trainer init info next to its - engine (dense `NCCLTrainerInitInfo`, `SparseNCCLTrainerInitInfo`) without - this shared module importing either. - """ - - master_address: str - master_port: int - world_size: int - - -def stateless_init_metadata_group( - master_address: str, - master_port: int, - rank: int, - world_size: int, - *, - listen_socket: socket.socket | None = None, -) -> "StatelessProcessGroup": - """Create only the TCPStore group used for metadata agreement.""" - from vllm.distributed.utils import StatelessProcessGroup - - return StatelessProcessGroup.create( - host=master_address, - port=master_port, - rank=rank, - world_size=world_size, - listen_socket=listen_socket, - ) - - -def stateless_init_process_group( - master_address: str, - master_port: int, - rank: int, - world_size: int, - device, -) -> "PyNcclCommunicator": - """VLLM provides `StatelessProcessGroup` to create a process group - without considering the global process group in torch.distributed. - It is recommended to create `StatelessProcessGroup`, and then initialize - the data-plane communication (NCCL) between external (train processes) - and vLLM workers. - """ - from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator - from vllm.distributed.utils import StatelessProcessGroup - - pg = StatelessProcessGroup.create( - host=master_address, port=master_port, rank=rank, world_size=world_size - ) - with unpinned_nccl_env(): - return PyNcclCommunicator(pg, device=device) - - -def uid_init_process_group( - nccl_unique_id_bytes: bytes, - rank: int, - world_size: int, - device, -) -> "PyNcclCommunicator": - """Join the NCCL group from pre-shared ``ncclUniqueId`` bytes. - - The torch-free rendezvous alternative to `stateless_init_process_group`: no - TCPStore, and therefore no barrier -- every rank must enter concurrently. - """ - from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator - - with unpinned_nccl_env(): - return PyNcclCommunicator.from_unique_id_bytes( - nccl_unique_id_bytes, - rank=rank, - world_size=world_size, - device=device, - ) - - -def _worker_group_rank(rank_offset: int, parallel_config: "ParallelConfig") -> int: - """Return this worker's rank in a trainer-to-worker process group.""" - dp_rank = parallel_config.data_parallel_index - world_size_per_dp = parallel_config.world_size # TP * PP - rank_within_dp = parallel_config.rank - return rank_offset + dp_rank * world_size_per_dp + rank_within_dp - - -def worker_init_metadata_group( - init_info: NCCLWeightTransferInitInfo, - parallel_config: "ParallelConfig", -) -> "StatelessProcessGroup": - """Join the trainer-to-worker TCP metadata group without constructing NCCL.""" - assert init_info.master_address is not None - assert init_info.master_port is not None - return stateless_init_metadata_group( - init_info.master_address, - init_info.master_port, - _worker_group_rank(init_info.rank_offset, parallel_config), - init_info.world_size, - ) - - -def worker_init_process_group( - init_info: NCCLWeightTransferInitInfo, - parallel_config: "ParallelConfig", -) -> "PyNcclCommunicator": - """Create the trainer<->worker NCCL group on an inference worker. - - Computes a unique rank for this worker across all data-parallel groups and - joins the trainer via whichever rendezvous mode `init_info` carries. - """ - rank = _worker_group_rank(init_info.rank_offset, parallel_config) - - device = torch.accelerator.current_device_index() - unique_id_bytes = init_info.nccl_unique_id_bytes - if unique_id_bytes is not None: - return uid_init_process_group( - unique_id_bytes, - rank, - init_info.world_size, - device, - ) - # __post_init__ guarantees the TCP pair when there is no unique id. - assert init_info.master_address is not None - assert init_info.master_port is not None - return stateless_init_process_group( - init_info.master_address, - init_info.master_port, - rank, - init_info.world_size, - device=device, - ) - - -def trainer_init( - init_info: NCCLRendezvous | dict, - rank: int = 0, -) -> "PyNcclCommunicator": - """Initialize NCCL process group for trainer-side weight transfer. - - Uses the current CUDA device (torch.accelerator.current_device_index()). - - Args: - init_info: Any object carrying the `NCCLRendezvous` fields (a trainer or - worker NCCL init info), or a dict with keys: - - master_address: str - - master_port: int - - world_size: int - rank: This trainer process's rank in the group. The broadcast backends - have a single trainer at rank 0; multi-rank trainers (m2n) occupy - ranks `[0, num_trainer_ranks)` and the workers start after them. - - Returns: - PyNcclCommunicator for weight transfer. - - """ - if isinstance(init_info, dict): - master_address = init_info["master_address"] - master_port = init_info["master_port"] - world_size = init_info["world_size"] - else: - master_address = init_info.master_address - master_port = init_info.master_port - world_size = init_info.world_size - - device = torch.accelerator.current_device_index() - return stateless_init_process_group( - master_address, - master_port, - rank, - world_size, - device, - ) diff --git a/vllm/distributed/weight_transfer/nccl_engine.py b/vllm/distributed/weight_transfer/nccl_engine.py deleted file mode 100644 index 331cb5eacc..0000000000 --- a/vllm/distributed/weight_transfer/nccl_engine.py +++ /dev/null @@ -1,461 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""NCCL-based (dense) weight transfer engine.""" - -from collections.abc import Iterator -from concurrent.futures import ThreadPoolExecutor -from dataclasses import asdict, dataclass -from typing import TYPE_CHECKING, ClassVar, Self - -import torch - -if TYPE_CHECKING: - from vllm.config import VllmConfig - from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator - -from vllm.config.weight_transfer import WeightTransferConfig -from vllm.distributed.weight_transfer.base import ( - ParamMeta, - TrainerInitInfo, - TrainerWeightTransferEngine, - VLLMWeightSyncClient, - WeightSource, - WeightTransferEngine, - WeightTransferUpdateInfo, -) -from vllm.distributed.weight_transfer.nccl_common import ( - NCCLWeightTransferInitInfo, - worker_init_payload, - worker_init_process_group, -) -from vllm.distributed.weight_transfer.nccl_common import ( - trainer_init as open_trainer_endpoint, -) -from vllm.distributed.weight_transfer.packed_tensor import ( - DEFAULT_PACKED_BUFFER_SIZE_BYTES, - DEFAULT_PACKED_NUM_BUFFERS, - packed_nccl_broadcast_consumer, - packed_nccl_broadcast_producer, -) - -# NCCLWeightTransferInitInfo is re-exported here for convenience; its canonical -# home is nccl_common, shared with the sparse backend. -__all__ = [ - "NCCLWeightTransferInitInfo", - "NCCLTrainerInitInfo", - "NCCLWeightTransferUpdateInfo", - "NCCLWeightTransferEngine", - "NCCLTrainerWeightTransferEngine", -] - - -@dataclass -class NCCLTrainerInitInfo(TrainerInitInfo): - """Trainer-side init info for the dense NCCL weight transfer backend. - - The sender opens its endpoint as NCCL rank 0, so it needs no `rank_offset`. - `world_size` is the full trainer+worker NCCL group size. `rank` (from - `TrainerInitInfo`) identifies this trainer process; rank 0 is the sender. - - The trainer joins over a TCPStore rendezvous (`master_address` + - `master_port`). Torch-free trainers (e.g. JAX) that cannot join a TCPStore - mint an `ncclUniqueId` themselves and drive rank 0 out of band; they ship a - `nccl_unique_id_b64` payload straight to the inference workers' - `init_weight_transfer_engine` (see `NCCLWeightTransferInitInfo`) rather than - going through this engine. - - `packed` / buffer sizes are the transfer's wire params. The trainer - propagates them to the worker at `trainer_init` so the two sides cannot - disagree. Note this defaults to packed, unlike the worker-side - `NCCLWeightTransferInitInfo`, whose default only applies when no trainer - ships a value. `backend` is the factory dispatch key.""" - - backend: ClassVar[str] = "nccl" - - master_address: str - master_port: int - world_size: int - packed: bool = True - packed_buffer_size_bytes: int = DEFAULT_PACKED_BUFFER_SIZE_BYTES - packed_num_buffers: int = DEFAULT_PACKED_NUM_BUFFERS - - -@dataclass -class NCCLWeightTransferUpdateInfo(WeightTransferUpdateInfo): - """Per-round update info for the dense NCCL weight transfer backend. - - Whether the transfer is packed (and the buffer geometry) is a must-agree - wire param carried on the init info (`NCCLTrainerInitInfo` / - `NCCLWeightTransferInitInfo`), not here; this carries only the per-round - parameter metadata. - """ - - names: list[str] - dtype_names: list[str] - shapes: list[list[int]] - - def __post_init__(self): - """Validate that all lists have the same length.""" - num_params = len(self.names) - if len(self.dtype_names) != num_params: - raise ValueError( - f"`dtype_names` should be of the same size as `names`: " - f"got {len(self.dtype_names)} and {len(self.names)}" - ) - if len(self.shapes) != num_params: - raise ValueError( - f"`shapes` should be of the same size as `names`: " - f"got {len(self.shapes)} and {len(self.names)}" - ) - - -class NCCLWeightTransferEngine( - WeightTransferEngine[NCCLWeightTransferInitInfo, NCCLWeightTransferUpdateInfo] -): - """Weight transfer engine using NCCL for communication between trainer and workers. - - This implementation uses NCCL broadcast operations to transfer dense - checkpoint-format weights from the trainer (rank 0) to all inference workers - in a process group. Received weights are loaded via the model's - `load_weights` using the layerwise reload lifecycle. - """ - - # Define backend-specific dataclass types - init_info_cls = NCCLWeightTransferInitInfo - update_info_cls = NCCLWeightTransferUpdateInfo - - def __init__( - self, - config: WeightTransferConfig, - vllm_config: "VllmConfig", - device: torch.device, - model: torch.nn.Module, - ) -> None: - super().__init__(config, vllm_config, device, model) - self.model_update_group: PyNcclCommunicator | None = None - # Set from the trainer-supplied init info at the handshake; defaults are - # only for the (unreachable) receive-before-init case. - self.packed = False - self.packed_buffer_size_bytes = DEFAULT_PACKED_BUFFER_SIZE_BYTES - self.packed_num_buffers = DEFAULT_PACKED_NUM_BUFFERS - - def init_transfer_engine(self, init_info: NCCLWeightTransferInitInfo) -> None: - """Initialize NCCL process group with the trainer and record the - trainer-supplied wire params so the worker decodes exactly as the - trainer encodes. - - Args: - init_info: NCCL initialization info containing master address, port, - rank offset, world size, and the packed wire params - - """ - self.packed = init_info.packed - self.packed_buffer_size_bytes = init_info.packed_buffer_size_bytes - self.packed_num_buffers = init_info.packed_num_buffers - self.model_update_group = worker_init_process_group( - init_info, self.parallel_config - ) - - def start_weight_update(self) -> None: - """Initialize layerwise reloading for the incoming checkpoint weights.""" - from vllm.model_executor.model_loader.reload import ( - initialize_layerwise_reload, - ) - - initialize_layerwise_reload(self.model) - - def finish_weight_update(self) -> None: - """Finalize layerwise reloading after all weights have been received.""" - from vllm.model_executor.model_loader.reload import ( - finalize_layerwise_reload, - ) - - finalize_layerwise_reload(self.model, self.model_config) - - def receive_weights(self, update_info: NCCLWeightTransferUpdateInfo) -> None: - """Receive weights from trainer via NCCL broadcast. - - Whether to use packed broadcasting (and the buffer geometry) is read - from `self.packed` / `self.packed_*`, set at the init handshake from the - trainer's init info, so it is guaranteed to match how the trainer - encoded. - - Args: - update_info: NCCL update info containing parameter names, dtypes, - and shapes - - """ - if self.model_update_group is None: - raise RuntimeError( - "NCCL weight transfer not initialized. " - "Call init_transfer_engine() first." - ) - - from vllm.model_executor.model_loader.mtp_validation import ( - disable_mtp_completeness_check, - ) - - with disable_mtp_completeness_check(): - if self.packed: - # Build iterator of (name, (shape, dtype)) from update_info - def state_dict_info_iterator(): - for name, dtype_name, shape in zip( - update_info.names, update_info.dtype_names, update_info.shapes - ): - dtype = getattr(torch, dtype_name) - yield (name, (shape, dtype)) - - packed_nccl_broadcast_consumer( - iterator=state_dict_info_iterator(), - group=self.model_update_group, - src=0, - post_unpack_func=self.model.load_weights, - buffer_size_bytes=self.packed_buffer_size_bytes, - num_buffers=self.packed_num_buffers, - device=self.device, - ) - else: - # Use simple one-by-one broadcasting - for name, dtype_name, shape in zip( - update_info.names, update_info.dtype_names, update_info.shapes - ): - dtype = getattr(torch, dtype_name) - weight = torch.empty(shape, dtype=dtype, device=self.device) - self.model_update_group.broadcast( - weight, src=0, stream=torch.cuda.current_stream() - ) - self.model.load_weights([(name, weight)]) - del weight - - def shutdown(self) -> None: - if self.model_update_group is not None: - self.model_update_group.destroy() - self.model_update_group = None - - -class NCCLTrainerWeightTransferEngine(TrainerWeightTransferEngine[NCCLTrainerInitInfo]): - """Trainer-side NCCL weight transfer engine. - - On the sender (rank 0) holds the NCCL communicator and drives the full - update round trip: it runs the inference-side `update_weights` concurrently - with the trainer-side broadcast (both rendezvous inside the same NCCL - calls), then finishes the update. Non-sender trainer ranks hold no - communicator; they only iterate the source to stay in the trainer-side - collective (e.g. FSDP `full_tensor()`) and skip the client RPCs and the - broadcast (all guarded on `is_sender`). - - `packed` / buffer sizes come from `NCCLTrainerInitInfo`; the sender - propagates them to the worker at `trainer_init` (on the worker-side init - info), so per-round payloads carry only parameter metadata. - """ - - init_info_cls = NCCLTrainerInitInfo - - def __init__( - self, - *, - client: VLLMWeightSyncClient, - source: WeightSource, - is_sender: bool = True, - packed: bool = True, - packed_buffer_size_bytes: int = DEFAULT_PACKED_BUFFER_SIZE_BYTES, - packed_num_buffers: int = DEFAULT_PACKED_NUM_BUFFERS, - ) -> None: - super().__init__(client=client, source=source, is_sender=is_sender) - self.packed = packed - self.packed_buffer_size_bytes = packed_buffer_size_bytes - self.packed_num_buffers = packed_num_buffers - self.model_update_group: PyNcclCommunicator | None = None - - @classmethod - def trainer_init( - cls, - init_info: NCCLTrainerInitInfo, - *, - client: VLLMWeightSyncClient, - source: WeightSource | None = None, - ) -> Self: - if source is None: - raise ValueError("NCCL trainer weight transfer requires a WeightSource.") - engine = cls( - client=client, - source=source, - is_sender=init_info.is_sender, - packed=init_info.packed, - packed_buffer_size_bytes=init_info.packed_buffer_size_bytes, - packed_num_buffers=init_info.packed_num_buffers, - ) - if not engine.is_sender: - # Non-sender trainer ranks aren't part of the transfer NCCL group and - # don't drive the inference side; they only participate in the - # trainer-side gather during send_weights. - return engine - - # Workers sit at rank_offset 1, after the single trainer sender rank 0. - worker_init_info = NCCLWeightTransferInitInfo( - master_address=init_info.master_address, - master_port=init_info.master_port, - rank_offset=1, - world_size=init_info.world_size, - packed=init_info.packed, - packed_buffer_size_bytes=init_info.packed_buffer_size_bytes, - packed_num_buffers=init_info.packed_num_buffers, - ) - - # The inference workers block inside init_weight_transfer_engine waiting - # for the NCCL rendezvous, so we kick that off on a side thread while we - # open the trainer endpoint (rank 0); both sides must rendezvous together. - with ThreadPoolExecutor(max_workers=1) as exe: - future = exe.submit( - engine.client.init_weight_transfer_engine, - worker_init_payload(worker_init_info), - ) - engine.model_update_group = open_trainer_endpoint(init_info) - future.result() # surface any inference-side init error - - return engine - - def send_weights(self) -> None: - assert self.source is not None # guaranteed by trainer_init / __init__ - source = self.source - - # Metadata is declared without gathering. For Megatron it is itself a - # collective, so every rank runs it; only the sender ships it. - meta = source.metadata() - - if not self.is_sender: - # Non-sender ranks only join the trainer-side gather collective. - self._broadcast(source, meta) - self._post_send_sync() - return - - update_info = NCCLWeightTransferUpdateInfo( - names=[m.name for m in meta], - dtype_names=[str(m.dtype).split(".")[-1] for m in meta], - shapes=[list(m.shape) for m in meta], - ) - - self.client.start_weight_update() - # update_weights (workers receive) must run concurrently with the - # trainer-side broadcast — both rendezvous inside the same NCCL calls. - exe = ThreadPoolExecutor(max_workers=1) - try: - future = exe.submit(self.client.update_weights, asdict(update_info)) - # Cheap best-effort: if update_weights already failed (e.g. a bad - # request rejected before any NCCL call), surface it now instead of - # hanging in broadcast waiting for a peer that will never arrive. - if future.done(): - future.result() - self._broadcast(source, meta) - future.result() # surface inference-side errors - finally: - # Never wait for the RPC thread here. If the broadcast raised, the - # worker is still blocked in the matching NCCL call and will never - # return, so joining would turn the error into a permanent hang. - # Let the exception out instead; the transfer group is unusable - # either way and the caller has to tear it down. (The orphaned - # thread is still joined at interpreter exit, so a caller that wants - # to exit cleanly after such a failure must not wait on it.) - exe.shutdown(wait=False) - self.client.finish_weight_update() - self._post_send_sync() - - def _broadcast(self, source: WeightSource, meta: list[ParamMeta]) -> None: - """Iterate the source (materializing each tensor — a collective on all - ranks) and, on the sender, broadcast from rank 0, packed or one-by-one. - Non-sender ranks only replay the iteration to stay in the collective.""" - if not self.is_sender: - for _ in source: - pass - return - - assert self.model_update_group is not None, ( - "trainer_init() must be called before _broadcast()." - ) - pairs = self._checked_iter(source, meta) - if self.packed: - packed_nccl_broadcast_producer( - iterator=pairs, - group=self.model_update_group, - src=0, - post_iter_func=lambda item: item[1], - buffer_size_bytes=self.packed_buffer_size_bytes, - num_buffers=self.packed_num_buffers, - ) - else: - stream = torch.cuda.current_stream() - for _name, tensor in pairs: - # NCCL sends `numel` elements straight from `data_ptr()`, so a - # non-contiguous view would ship whatever follows its base - # pointer. Keep the copy referenced until the broadcast is - # enqueued. (The packed path linearizes in `pack_tensors`.) - send = tensor if tensor.is_contiguous() else tensor.contiguous() - self.model_update_group.broadcast(send, src=0, stream=stream) - - @staticmethod - def _checked_iter( - source: WeightSource, meta: list[ParamMeta] - ) -> Iterator[tuple[str, torch.Tensor]]: - """Yield the source's pairs, checking each against what the worker was - told to expect. - - The worker sizes its receive buffers — and in packed mode cuts its chunk - boundaries — from the update info, which is built from `metadata()`. If - iteration disagrees with it, the two sides split the stream differently - and the transfer hangs in NCCL or loads garbage. Checking here costs one - comparison per parameter and turns that into an error naming the first - divergent parameter. Sender-only: under pipeline parallelism a - non-sender's yielded tensor is not meaningful. - """ - sent = 0 - for name, tensor in source: - if sent >= len(meta): - raise ValueError( - f"WeightSource yielded more parameters than metadata() " - f"declared ({len(meta)}); first extra is {name!r}." - ) - expected = meta[sent] - if ( - name != expected.name - or tensor.dtype != expected.dtype - or tuple(tensor.shape) != expected.shape - ): - raise ValueError( - "WeightSource metadata() disagrees with iteration at index " - f"{sent}: declared {expected.name!r} " - f"{expected.dtype} {tuple(expected.shape)}, got {name!r} " - f"{tensor.dtype} {tuple(tensor.shape)}. Both channels must " - "enumerate the same parameters in the same order." - ) - sent += 1 - yield name, tensor - if sent != len(meta): - raise ValueError( - f"WeightSource yielded {sent} parameters but metadata() " - f"declared {len(meta)}; the worker is waiting for the rest." - ) - - def _post_send_sync(self) -> None: - """Wait for this rank's transfer work to land before returning. - - Broadcasts are only *enqueued* by `send_weights`: the unpacked path on - the current stream, the packed path on the producer's own streams (which - it drains itself). Waiting here lets a caller mutate parameters, or - start the next step on another stream, as soon as `send_weights` - returns, instead of silently depending on same-stream ordering. Every - rank waits: a non-sender's `full_tensor()` gathers feed the sender's - broadcast, so they must have landed before it may touch its shards. - - Unlike IPC there is no cross-rank barrier here. Nothing in this backend - outlives the collective it travelled in (IPC's barrier keeps shared - buffers alive until every consumer has opened them), so a barrier would - only add a dependency on the default process group that this backend - otherwise does not have. - """ - if torch.cuda.is_available(): - torch.cuda.current_stream().synchronize() - - def shutdown(self) -> None: - if self.model_update_group is not None: - self.model_update_group.destroy() - self.model_update_group = None diff --git a/vllm/distributed/weight_transfer/packed_tensor.py b/vllm/distributed/weight_transfer/packed_tensor.py deleted file mode 100644 index 2e71538380..0000000000 --- a/vllm/distributed/weight_transfer/packed_tensor.py +++ /dev/null @@ -1,494 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Packed tensor utilities for efficient weight transfer.""" - -import math -from collections.abc import Callable, Iterator -from dataclasses import dataclass -from functools import cache -from typing import Any - -import torch -from torch.multiprocessing.reductions import reduce_tensor - -# Default values for packed tensor transfer. -DEFAULT_PACKED_BUFFER_SIZE_BYTES = 1024 * 1024 * 1024 # 1GB -DEFAULT_PACKED_NUM_BUFFERS = 2 - - -# Streams are cached across calls: the caching allocator keys cached blocks by the -# stream they were allocated on, so fresh streams at every call would strand the -# packed buffers' reserved memory per call. See gh-52950. -@cache -def _get_streams(device_idx: int, num_buffers: int) -> tuple[torch.cuda.Stream, ...]: - return tuple(torch.cuda.Stream(device=device_idx) for _ in range(num_buffers)) - - -def unpack_tensor( - packed_tensor: torch.Tensor, - names: list[str], - shapes: list[list[int]], - dtypes: list[torch.dtype], - tensor_sizes: list[int], -) -> list[tuple[str, torch.Tensor]]: - """Unpack a packed uint8 tensor into a list of named tensors. - - The returned tensors are **views** of ``packed_tensor`` (the - ``.contiguous()`` call is a no-op on already-contiguous row-slices). - If ``packed_tensor`` lives in storage that may be reused — e.g. a - reused CUDA IPC buffer — callers must clone the results before the - underlying storage is overwritten. - - Args: - packed_tensor: The packed torch.uint8 tensor to unpack - names: List of tensor names - shapes: List of tensor shapes - dtypes: List of tensor dtypes - tensor_sizes: List of tensor sizes in bytes - - """ - unpacked_tensors = packed_tensor.split(tensor_sizes) - - return [ - (name, tensor.contiguous().view(dtype).view(shape)) - for name, shape, dtype, tensor in zip(names, shapes, dtypes, unpacked_tensors) - ] - - -@dataclass -class PackedChunk: - """Result of packing tensors into a single contiguous uint8 buffer.""" - - packed_tensor: torch.Tensor - names: list[str] - shapes: list[list[int]] - dtypes: list[torch.dtype] - tensor_sizes: list[int] - - -def pack_tensors( - iterator: Iterator[tuple[str, torch.Tensor]], - post_iter_func: Callable[[tuple[str, torch.Tensor]], torch.Tensor], - buffer_size_bytes: int, - tensor_list: list[torch.Tensor] | None = None, - current_size: int = 0, -) -> PackedChunk | None: - """Pack tensors from an iterator into a single contiguous uint8 buffer. - - Consumes from the iterator until the accumulated size exceeds - buffer_size_bytes or the iterator is exhausted, then returns a - PackedChunk. Returns None if no tensors were consumed. - - Args: - iterator: Iterator of (name, tensor) pairs - post_iter_func: Applied to each item before linearizing to uint8 - buffer_size_bytes: Max bytes before flushing - tensor_list: Pre-existing tensor list to append to (for NCCL - multi-buffer reuse). If None, a fresh list is created. - current_size: Byte count already accumulated in tensor_list - - """ - if tensor_list is None: - tensor_list = [] - - names: list[str] = [] - shapes: list[list[int]] = [] - dtypes: list[torch.dtype] = [] - tensor_sizes: list[int] = [] - total_bytes = current_size - - while True: - try: - item = next(iterator) - except StopIteration: - break - - name, orig_tensor = item - # Apply post processing and convert to linearized uint8 tensor - tensor = post_iter_func(item).contiguous().view(-1).view(torch.uint8) - - if tensor.numel() > buffer_size_bytes: - import warnings - - warnings.warn( - f"Tensor '{name}' has size {tensor.numel()} bytes, which " - f"exceeds buffer_size_bytes={buffer_size_bytes}.", - stacklevel=2, - ) - - tensor_list.append(tensor) - names.append(name) - shapes.append(list(orig_tensor.shape)) - dtypes.append(orig_tensor.dtype) - tensor_sizes.append(tensor.numel()) - total_bytes += tensor.numel() - - if total_bytes > buffer_size_bytes: - break - - if not tensor_list: - return None - - packed = torch.cat(tensor_list, dim=0) - del tensor_list - return PackedChunk( - packed_tensor=packed, - names=names, - shapes=shapes, - dtypes=dtypes, - tensor_sizes=tensor_sizes, - ) - - -# ── NCCL packed broadcast ────────────────────────────────────────────── - - -def packed_nccl_broadcast_producer( - iterator: Iterator[tuple[str, torch.Tensor]], - group: Any, - src: int, - post_iter_func: Callable[[tuple[str, torch.Tensor]], torch.Tensor], - buffer_size_bytes: int = DEFAULT_PACKED_BUFFER_SIZE_BYTES, - num_buffers: int = DEFAULT_PACKED_NUM_BUFFERS, -) -> None: - """Broadcast tensors in a packed manner from trainer to workers. - - Args: - iterator: Iterator of model parameters. Returns a tuple of (name, tensor) - group: Process group (PyNcclCommunicator) - src: Source rank (0 in current implementation) - post_iter_func: Function to apply to each (name, tensor) pair before - packing, should return a tensor - buffer_size_bytes: Size in bytes for each packed tensor buffer. - Both producer and consumer must use the same value. - num_buffers: Number of buffers for double/triple buffering. - Both producer and consumer must use the same value. - - """ - streams = _get_streams(torch.accelerator.current_device_index(), num_buffers) - # Keep references to in-flight chunks so their packed_tensors - # aren't freed while an async broadcast is still reading them. - in_flight: list[PackedChunk | None] = [None] * num_buffers - buffer_idx = 0 - - while True: - # Synchronize the current stream - streams[buffer_idx].synchronize() - # Previous chunk on this buffer slot is now safe to free - in_flight[buffer_idx] = None - # Start tasks for the new buffer in a new stream - with torch.cuda.stream(streams[buffer_idx]): - chunk = pack_tensors(iterator, post_iter_func, buffer_size_bytes) - if chunk is None: - break - # Pack the tensors and call broadcast collective - group.broadcast(chunk.packed_tensor, src=src) - # Hold reference until this stream is synchronized - in_flight[buffer_idx] = chunk - # Move to the next buffer - buffer_idx = (buffer_idx + 1) % num_buffers - - # Drain every slot before returning. - for stream in streams: - stream.synchronize() - - -def packed_nccl_broadcast_consumer( - iterator: Iterator[tuple[str, tuple[list[int], torch.dtype]]], - group: Any, - src: int, - post_unpack_func: Callable[[list[tuple[str, torch.Tensor]]], None], - buffer_size_bytes: int = DEFAULT_PACKED_BUFFER_SIZE_BYTES, - num_buffers: int = DEFAULT_PACKED_NUM_BUFFERS, - device: torch.device | str = "cuda", -) -> None: - """Consume packed tensors and unpack them into a list of tensors. - - Args: - iterator: Iterator of parameter metadata. Returns (name, (shape, dtype)) - group: Process group (PyNcclCommunicator) - src: Source rank (0 in current implementation) - post_unpack_func: Function to apply to each list of (name, tensor) after - unpacking - buffer_size_bytes: Size in bytes for each packed tensor buffer. - Both producer and consumer must use the same value. - num_buffers: Number of buffers for double/triple buffering. - Both producer and consumer must use the same value. - device: Device for the receive buffers. Must match the device the NCCL - communicator was created on (the worker's assigned device). - - """ - target_packed_tensor_size = buffer_size_bytes - - streams = _get_streams(torch.accelerator.current_device_index(), num_buffers) - buffer_idx = 0 - - packing_tensor_meta_data: list[list[tuple[str, list[int], torch.dtype, int]]] = [ - [] for _ in range(num_buffers) - ] - packing_tensor_sizes: list[int] = [0 for _ in range(num_buffers)] - packed_tensors: list[torch.Tensor] = [ - torch.empty(0, dtype=torch.uint8, device=device) for _ in range(num_buffers) - ] - - while True: - # Synchronize the current stream - streams[buffer_idx].synchronize() - with torch.cuda.stream(streams[buffer_idx]): - # Initialize the packing tensor meta data - packing_tensor_meta_data[buffer_idx] = [] - packing_tensor_sizes[buffer_idx] = 0 - try: - # Form a packed tensor - while True: - name, (shape, dtype) = next(iterator) - tensor_size = math.prod(shape) * dtype.itemsize - packing_tensor_meta_data[buffer_idx].append( - (name, shape, dtype, tensor_size) - ) - packing_tensor_sizes[buffer_idx] += tensor_size - if packing_tensor_sizes[buffer_idx] > target_packed_tensor_size: - break - # Create a packed tensor and broadcast it - packed_tensors[buffer_idx] = torch.empty( - packing_tensor_sizes[buffer_idx], dtype=torch.uint8, device=device - ) - group.broadcast(packed_tensors[buffer_idx], src=src) - # Load the packed tensor into the model - names, shapes, dtypes, tensor_sizes = zip( - *packing_tensor_meta_data[buffer_idx] - ) - post_unpack_func( - unpack_tensor( - packed_tensors[buffer_idx], - list(names), - list(shapes), - list(dtypes), - list(tensor_sizes), - ) - ) - # Move to the next buffer - buffer_idx = (buffer_idx + 1) % num_buffers - except StopIteration: - # Do the last broadcast if there are remaining tensors - if len(packing_tensor_meta_data[buffer_idx]) > 0: - # Create a packed tensor and broadcast it - packed_tensors[buffer_idx] = torch.empty( - packing_tensor_sizes[buffer_idx], - dtype=torch.uint8, - device=device, - ) - group.broadcast(packed_tensors[buffer_idx], src=src) - # Load the packed tensor into the model - names, shapes, dtypes, tensor_sizes = zip( - *packing_tensor_meta_data[buffer_idx] - ) - post_unpack_func( - unpack_tensor( - packed_tensors[buffer_idx], - list(names), - list(shapes), - list(dtypes), - list(tensor_sizes), - ) - ) - break - - # Drain every slot before returning. - for stream in streams: - stream.synchronize() - - -# ── IPC packed transfer ──────────────────────────────────────────────── - - -@dataclass -class PackedIpcChunk: - """Metadata and IPC handle for a single packed chunk.""" - - names: list[str] - shapes: list[list[int]] - dtype_names: list[str] - tensor_sizes: list[int] - ipc_handle: dict[str, tuple] - - -def packed_ipc_producer( - iterator: Iterator[tuple[str, torch.Tensor]], - gpu_uuid: str, - post_iter_func: Callable[[tuple[str, torch.Tensor]], torch.Tensor], - buffer_size_bytes: int = DEFAULT_PACKED_BUFFER_SIZE_BYTES, -) -> Iterator[PackedIpcChunk]: - """Pack tensors into a reusable IPC buffer and yield handles. - - Allocates a single GPU buffer of ``buffer_size_bytes`` and registers - it for IPC once via ``reduce_tensor``. Each chunk's packed data is - copied into this buffer before yielding, so only one IPC-shared - allocation exists for the lifetime of the transfer. - - Callers **must** ensure the consumer has finished reading the buffer - (e.g. ``ray.get`` returned) before resuming the generator for the - next chunk. - - Args: - iterator: Iterator of (name, tensor) pairs. - gpu_uuid: Physical GPU UUID string for this rank. - post_iter_func: Applied to each (name, tensor) before packing. - buffer_size_bytes: Exact capacity of the reusable IPC buffer. - Every chunk is guaranteed to fit within this size. A - ``ValueError`` is raised if any single tensor exceeds it. - - """ - ipc_buffer = torch.empty(buffer_size_bytes, dtype=torch.uint8, device="cuda") - _, ipc_args = reduce_tensor(ipc_buffer) - - names: list[str] = [] - shapes: list[list[int]] = [] - dtypes: list[torch.dtype] = [] - tensor_sizes: list[int] = [] - total_bytes = 0 - - for name, orig_tensor in iterator: - flat = ( - post_iter_func((name, orig_tensor)).contiguous().view(-1).view(torch.uint8) - ) - - if flat.numel() > buffer_size_bytes: - raise ValueError( - f"Tensor '{name}' has size {flat.numel()} bytes, " - f"which exceeds buffer_size_bytes={buffer_size_bytes}. " - f"Increase buffer_size_bytes to at least {flat.numel()}." - ) - - if total_bytes and total_bytes + flat.numel() > buffer_size_bytes: - # Drain queued copies so the consumer sees a fully-written buffer. - torch.cuda.current_stream().synchronize() - yield PackedIpcChunk( - names=names, - shapes=shapes, - dtype_names=[str(d).split(".")[-1] for d in dtypes], - tensor_sizes=tensor_sizes, - ipc_handle={gpu_uuid: ipc_args}, - ) - # Rebind to fresh lists so the yielded chunk's metadata is - # not mutated while the consumer is still reading. - names, shapes, dtypes, tensor_sizes = [], [], [], [] - total_bytes = 0 - - ipc_buffer[total_bytes : total_bytes + flat.numel()].copy_(flat) - names.append(name) - shapes.append(list(orig_tensor.shape)) - dtypes.append(orig_tensor.dtype) - tensor_sizes.append(flat.numel()) - total_bytes += flat.numel() - - if total_bytes: - torch.cuda.current_stream().synchronize() - yield PackedIpcChunk( - names=names, - shapes=shapes, - dtype_names=[str(d).split(".")[-1] for d in dtypes], - tensor_sizes=tensor_sizes, - ipc_handle={gpu_uuid: ipc_args}, - ) - - -class PackedBufferImporter: - """One-slot cache for the consumer-side mapping of a packed IPC buffer. - - torch's cross-process refcount pairs one ``reduce_tensor`` export with - one consumer rebuild-release cycle. The producer exports its buffer once - and ships the same args with every chunk, so rebuilding and releasing - per chunk decrements the counter once per chunk, drives it negative, - and the dropped buffer is then never reclaimable by ``ipc_collect`` on - the producer side (it leaks one buffer per transfer). Caching the - rebuilt buffer keyed by the rebuild args restores the pairing: chunks - of one transfer reuse the mapping (it aliases producer memory, so it - always reads the current chunk's bytes), and replacing the entry when - the next export arrives — or ``close()`` — releases the previous - mapping exactly once. - - The importer that consumes a multi-chunk transfer must be one object - reused across chunks; a fresh importer per chunk reintroduces the - per-chunk release. ``IPCWeightTransferEngine`` owns one instance per - engine and closes it on shutdown. - """ - - def __init__(self) -> None: - self._entry: tuple[tuple, torch.Tensor] | None = None - - def rebuild(self, list_args: list) -> torch.Tensor: - from torch.multiprocessing.reductions import rebuild_cuda_tensor - - key = tuple(list_args) - if self._entry is not None and self._entry[0] == key: - return self._entry[1] - packed = rebuild_cuda_tensor(*list_args) - self._entry = (key, packed) - return packed - - def close(self) -> None: - """Release the held mapping (one producer-side refcount decrement).""" - self._entry = None - - -def packed_ipc_consumer( - ipc_handle: dict[str, tuple], - names: list[str], - shapes: list[list[int]], - dtype_names: list[str], - tensor_sizes: list[int], - device_index: int, - importer: "PackedBufferImporter | None" = None, -) -> list[tuple[str, torch.Tensor]]: - """Unpack a single packed IPC chunk into named tensors. - - Reconstructs the packed buffer via rebuild_cuda_tensor, unpacks - into individual tensors, and clones each into independent storage - before returning. - - The clone is intentional: the producer reuses one IPC buffer across - chunks, so any tensor view that aliases the buffer would observe the - *next* chunk's bytes as soon as the producer's generator is resumed. - Callers that retain references past their own update_weights call - (notably vLLM's layerwise reload, which buffers ``bound_args`` for - replay in ``_layerwise_process``) would otherwise replay against - stale data and silently corrupt multi-chunk weight transfers. - - Args: - ipc_handle: Mapping of GPU UUID to rebuild_cuda_tensor args tuple - names: Parameter names in the packed buffer - shapes: Parameter shapes - dtype_names: Parameter dtype name strings (e.g. "float16") - tensor_sizes: Size in bytes of each parameter in the packed buffer - device_index: Local CUDA device index - importer: Import cache that must be shared across all chunks of one - export so the buffer mapping is opened and released exactly once - (see ``PackedBufferImporter``). ``None`` builds an ephemeral one, - which is only safe for single-chunk consumption. - - """ - props = torch.cuda.get_device_properties(device_index) - physical_gpu_id = str(props.uuid) - - if physical_gpu_id not in ipc_handle: - raise ValueError( - f"IPC handle not found for GPU UUID {physical_gpu_id}. " - f"Available UUIDs: {list(ipc_handle.keys())}" - ) - - args = ipc_handle[physical_gpu_id] - list_args = list(args) - list_args[6] = device_index - if importer is None: - importer = PackedBufferImporter() - packed = importer.rebuild(list_args) - - content_size = sum(tensor_sizes) - packed = packed[:content_size] - - dtypes = [getattr(torch, dn) for dn in dtype_names] - return [ - (name, t.clone()) - for name, t in unpack_tensor(packed, names, shapes, dtypes, tensor_sizes) - ] diff --git a/vllm/distributed/weight_transfer/sharded_rdt_common.py b/vllm/distributed/weight_transfer/sharded_rdt_common.py deleted file mode 100644 index 0d91362817..0000000000 --- a/vllm/distributed/weight_transfer/sharded_rdt_common.py +++ /dev/null @@ -1,352 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Shared pieces of the sharded-RDT backend: the op-chain allowlist, buffer -sizing and the minimum Ray version, plus ``RdtRouter`` — consumer-only, since -routing is a consumer decision. - -The gather-group partition itself is ``base.layerwise_groups``: it defines what a -group index means for any ``WeightSource``, not just this transport. -""" - -from collections.abc import Callable -from typing import Any - -import torch - -# The op-chain contract, with two enforcers. The consumer's ``FakeRDTTensor`` -# intercepts exactly these methods during the bake and records the string name; -# anything else reaches ``__torch_dispatch__`` and raises, so a loader needing -# real data fails at init rather than transferring wrong bytes. The producer -# replays a chain as ``getattr(tensor, op)(...)`` and refuses anything outside -# ``ALLOWED_OPS``, so a spoofed consumer cannot invoke arbitrary methods. -# -# Every entry must be a pure view; ``to`` in particular must never be added, as -# it is the dtype/device escape the bake exists to reject. Both sets derive from -# this one table so they cannot disagree. -SUPPORTED_OPS: dict[Callable, str] = { - torch.Tensor.narrow: "narrow", - torch.Tensor.view: "view", - torch.Tensor.reshape: "reshape", - torch.Tensor.__getitem__: "__getitem__", - torch.Tensor.unsqueeze: "unsqueeze", - torch.Tensor.squeeze: "squeeze", - torch.Tensor.transpose: "transpose", - torch.Tensor.t: "t", - torch.Tensor.permute: "permute", - torch.Tensor.flatten: "flatten", - torch.Tensor.contiguous: "contiguous", - torch.Tensor.chunk: "chunk", - # Multi-return like chunk: one child per output, trailing __getitem__(i). - torch.Tensor.unbind: "unbind", -} - -ALLOWED_OPS = frozenset(SUPPORTED_OPS.values()) - -# Ray Direct Transport landed in halves: the actor opt-in -# ``enable_tensor_transport`` in 2.49, and ``ray.experimental``'s -# ``register_nixl_memory`` / ``set_target_for_ref`` — which this transport calls -# directly — in 2.55. The floor is the tested version rather than the earliest -# one that could import; 2.55 has never been run against. -RDT_MIN_RAY_VERSION = "2.56.0" - - -def check_ray_rdt_version() -> None: - """Refuse an installed Ray older than the one this backend is tested on. - - vLLM does not depend on Ray, so there is no pin to carry this. Without the - check the failure is an opaque option-validation error out of - ``.options(enable_tensor_transport=True)`` (below 2.49) or an ImportError - raised deep in the first pull, long after init reported success (below 2.55). - - Raises: - ValueError: the installed Ray predates ``RDT_MIN_RAY_VERSION``. - - """ - import importlib.metadata - - from packaging import version - - required = version.parse(RDT_MIN_RAY_VERSION) - current = version.parse(importlib.metadata.version("ray")) - if current < required: - raise ValueError( - f"The 'sharded_rdt' weight transfer backend requires Ray " - f">= {required}, the version it is tested against; Ray Direct " - f"Transport (ray.experimental.register_nixl_memory / " - f"set_target_for_ref) needs at least 2.55. Found {current}. " - f"Run `pip install -U 'ray>={required}'`." - ) - - -def register_nixl_memory(tensor: torch.Tensor) -> None: - """Register a tensor with Ray using the platform's NIXL package.""" - from vllm.distributed.nixl_utils import alias_nixl_for_ray - - alias_nixl_for_ray() - from ray.experimental import register_nixl_memory as ray_register_nixl_memory - - ray_register_nixl_memory(tensor) - - -def assign_producer_indices( - num_producers: int, num_consumers: int, consumer_idx: int -) -> list[int]: - """Producers (global indices) that consumer ``consumer_idx`` binds.""" - p = max(1, num_producers) - c = max(1, num_consumers) - if p >= c: - return list(range(consumer_idx * p // c, (consumer_idx + 1) * p // c)) - return [consumer_idx * p // c] - - -class RdtRouter: - """Decides which producer serves each weight name, and — once bound — sends. - - Ownership is per NAME. ``owner_sets`` holds the few distinct producer sets - that occur, and ``name_owner_class[i]`` indexes it for ``names[i]``; a name - determines both its owner set and its gather group, so neither appears in the - routing API. Empty tables mean every producer holds everything. - - Any placement is expressible this way: a pipeline stage is a set of names - sharing one owner set, an expert is a name whose owner set is a single rank, - and a group produced by two stages is just two classes inside one group. - - Both engines derive the same tables from the same wire data, so they agree on - who serves what. Disagreement is not a wrong answer but a hang or a loud - misroute: a pull sent to a producer that never gathered the name trips its - served-names guard. - - Routing is consumer-only. The trainer publishes what it holds and answers - whatever arrives; it never asks who serves what. - """ - - def __init__( - self, - num_producers: int, - num_consumers: int, - owner_sets: list[list[int]] | None = None, - name_owner_class: list[int] | None = None, - names: list[str] | None = None, - group_lens: list[int] | None = None, - workers_per_replica: int = 0, - ) -> None: - """Build the routing tables from the wire data both engines receive. - - The four table arguments travel together and default together: with all - of them empty the router degrades to "every producer holds everything". - - Args: - num_producers: Trainer ranks. Owner indices are positions in this - range, and ``validate`` rejects any that fall outside it. - num_consumers: Inference workers across the whole fleet. Fixes - the id space; the block carve uses ``workers_per_replica`` - of it. - owner_sets: The distinct producer sets that occur, one row per - owner class; each row is deduplicated and sorted here. Empty - means a single class owning every producer. - name_owner_class: Parallel to ``names`` — ``name_owner_class[i]`` - indexes ``owner_sets`` for ``names[i]``. A name with no entry - falls back to class 0. - names: Every parameter name, in group-major order: concatenating - the gather groups reproduces this list exactly. The other two - tables are keyed by this order. - group_lens: Length of each gather group, consecutive over - ``names``, so they sum to ``len(names)``. Fixes name -> group - and the per-group owner union the free barrier fans out to. - workers_per_replica: Consumers per inference DEPLOYMENT. The block - carve spreads this many consumers over an owner set and every - deployment reuses that carve, so the same worker of each - deployment resolves one producer (see ``producer_for``). 0 - means one deployment: carve over the whole fleet. - - Raises: - ValueError: ``workers_per_replica`` does not divide - ``num_consumers``, so the deployments are not uniform and two - workers of one deployment would share a block index. - - """ - self.num_producers = max(1, num_producers) - self.num_consumers = max(1, num_consumers) - # Width of the block carve: one deployment's worth of consumers, so - # deployments overlay rather than spread. See ``producer_for``. - self._block_consumers = min( - self.num_consumers, max(1, workers_per_replica or self.num_consumers) - ) - if self.num_consumers % self._block_consumers: - raise ValueError( - f"workers_per_replica={workers_per_replica} does not divide " - f"num_consumers={self.num_consumers}; the inference deployments " - f"must be uniform for the per-deployment block carve to be " - f"well defined." - ) - self._owner_sets = ( - [sorted(set(row)) for row in owner_sets] - if owner_sets - else [list(range(self.num_producers))] - ) - names = list(names or []) - classes = list(name_owner_class or []) - self._class_of = { - n: (classes[i] if i < len(classes) else 0) for i, n in enumerate(names) - } - # name -> gather group, and the per-group owner union the free barrier - # fans out to. Both are pure functions of the tables, so precompute once. - self._group_of: dict[str, int] = {} - self._group_owners: list[list[int]] = [] - pos = 0 - for gi, glen in enumerate(group_lens or []): - union: set[int] = set() - for n in names[pos : pos + glen]: - self._group_of[n] = gi - c = self._class_of.get(n, 0) - # Bounds-checked: a bad class must surface from validate() as a - # named ValueError, not as an IndexError from this precompute. - if 0 <= c < len(self._owner_sets): - union.update(self._owner_sets[c]) - pos += glen - self._group_owners.append(sorted(union)) - # Bound at init on the consumer; the rule methods never touch these. - self._actors: list[Any] = [] - self._produce_methods: list[Any] = [] - self.consumer_id = 0 - - # ---------------- rules (pure; safe to use unbound) ---------------- - - def class_of(self, name: str) -> int: - """The name's owner class — the planner's bucketing key, since all names - of a chunk must share one producer.""" - return self._class_of.get(name, 0) - - def owners(self, name: str) -> list[int]: - """Every producer holding ``name``.""" - return list(self._owner_sets[self.class_of(name)]) - - def group_owners(self, group_idx: int) -> list[int]: - """Every producer holding any name of ``group_idx``. - - The one group-keyed rule, because the free barrier is per group: a - consumer signals ``free_group(gi)`` at each of these and the producer - counts signals against its live-consumer total. Names route; groups free. - """ - if not self._group_owners: - return list(range(self.num_producers)) - return list(self._group_owners[group_idx]) - - def producer_for(self, consumer_id: int, name: str) -> int: - """The single producer ``consumer_id`` pulls ``name`` from. - - Blocks consumers across the name's owner set with the same rule that - binds producers globally, then rotates by the name's GROUP index so a - consumer spreads its groups over its block instead of hammering one NIC. - Rotating per group (not per name) keeps every name of a chunk on one - producer, which is what lets a chunk be a single pull. - - The carve is over ONE DEPLOYMENT (this consumer's index within its own - deployment, over ``workers_per_replica`` of them), so the same worker of - every deployment resolves the same producer for every name. Carving over - the whole fleet instead spreads the multi-owner names of different - deployments onto different producers while single-owner names still - route by ownership, so a producer serves several distinct worker - indices, each needing its own serve ring, and none of them can share a - slot. With one deployment the width is the fleet, so this is the plain - block rule. - """ - own = self.owners(name) - if not own: - raise ValueError(f"{name!r} has no owner") - block = assign_producer_indices( - len(own), self._block_consumers, consumer_id % self._block_consumers - ) - return own[block[self._group_of.get(name, 0) % len(block)]] - - def validate(self) -> None: - """Check the ownership tables can be served. - - Raises: - ValueError: an owner set is empty or out of range, or a class index - does not resolve. - - """ - for c, row in enumerate(self._owner_sets): - if not row: - raise ValueError(f"owner set {c} is empty: no producer holds it") - bad = [p for p in row if not 0 <= p < self.num_producers] - if bad: - raise ValueError(f"owner set {c} out of range: {bad}") - for name, c in self._class_of.items(): - if not 0 <= c < len(self._owner_sets): - raise ValueError( - f"{name!r} has owner class {c}, but only " - f"{len(self._owner_sets)} owner set(s) were shipped" - ) - - # ---------------- send surface (needs the binding) ---------------- - - def bind(self, actors: list, produce_methods: list, consumer_id: int) -> None: - """Attach this consumer's producer handles. Rebuilt wholesale per init, - never appended to: every owner index is a position in these lists, so a - rejoining engine that re-inits must not shift them.""" - if len(actors) != len(produce_methods): - raise ValueError("actors and produce_methods must be parallel") - self._actors = list(actors) - self._produce_methods = list(produce_methods) - self.consumer_id = consumer_id - - def _bound(self) -> None: - if not self._produce_methods: - raise RuntimeError("RdtRouter has no producers bound; call bind() first.") - - def pull(self, owner: int, keys: list, seq: int) -> Any: - """Issue one packed pull to ``owner``. - - The consumer id keys the producer's serve ring; without it every worker - is served out of ring 0 and concurrent pulls overwrite each other's blob. - ``seq`` is this call's index in the stream to ``owner`` (see ``_Chunk``): - it selects the serve slot in ISSUE order, so a slot is never repacked - while the read of its previous contents is still in flight. - """ - self._bound() - return self._produce_methods[owner].remote( - keys, consumer_id=self.consumer_id, seq=seq - ) - - def free_group(self, group_idx: int) -> list: - """Signal the group done at every producer holding any of its names.""" - self._bound() - return [ - self._actors[p].free_group.remote(group_idx) - for p in self.group_owners(group_idx) - ] - - def reserve_serve_buffers( - self, bytes_by_producer: list[int], plan_digests: list[str] | None = None - ) -> list: - """Ask each producer to pre-register a serve ring sized to the most this - consumer will pull from it. - - ``plan_digests[p]`` describes the chunks this consumer pulls from - producer ``p``, in pull order. A producer that shares one serve ring - across the consumers of several deployments compares it across them, so - a fleet whose deployments are not identical fails at init instead of - stalling mid-sync. A producer that shares nothing ignores it. - """ - self._bound() - digests: list = ( - list(plan_digests) if plan_digests else [None] * len(bytes_by_producer) - ) - return [ - self._actors[p].reserve_serve_buffer.remote( - self.consumer_id, nb, digests[p] - ) - for p, nb in enumerate(bytes_by_producer) - if nb > 0 - ] - - -def buffer_alloc_bytes(nbytes: int, presize: int = 0) -> int: - """Size a NIXL buffer / ring slot for ``nbytes``: the max of the request, an - optional ``presize`` floor, and a coarse 256MB round-up, so the buffer is - allocated ONCE and never regrows. Regrowth is a correctness hazard, not just a - perf one -- see ``buffer_presize_gb``. Shared by the consumer's receive buffers - and the producer's serve rings.""" - return max(nbytes, presize, -(-nbytes // (256 << 20)) * (256 << 20)) diff --git a/vllm/distributed/weight_transfer/sharded_rdt_engine.py b/vllm/distributed/weight_transfer/sharded_rdt_engine.py deleted file mode 100644 index 2293be3ccd..0000000000 --- a/vllm/distributed/weight_transfer/sharded_rdt_engine.py +++ /dev/null @@ -1,1477 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Sharded Ray Direct Transport (RDT) weight transfer engine (consumer side). - -Pulls only the slice each vLLM worker consumes under tensor/expert parallelism, -not the full HF-format tensor. - -Two phases. BAKE, once at ``init_transfer_engine``: drive ``model.load_weights`` -against ``FakeRDTTensor`` placeholders and record, per leaf module, how each -destination slice is fetched (an op chain) and where it lands (an ``as_strided`` -descriptor). REPLAY, every sync: no ``load_weights``, no ``FakeRDTTensor`` -dispatch, no discovery — pull the recorded slices in packed chunks over a ring -of receive buffers, scatter them into freshly materialized params, then quant -and kernel-copy. A live name with no recorded plan fails the plan build: there -is no fallback load. - -Weights arrive in checkpoint format: the engine drives layerwise reload itself, -in ``start_weight_update`` / ``finish_weight_update``. - -Data flow ---------- -One thing at four resolutions, over three lifetimes. ``FetchKey`` -- -``(name, op_chain)``, "which slice of which trainer tensor" -- is the atom; -everything else is bookkeeping around it. - - BAKE once at init, kept for the engine's life - FakeRDTTensor intercepts the model's loaders; each copy_ records a - _Scatter (sharded_rdt_fake): src FetchKey, owning layer, the destination - as_strided region, and the produced dtype/nbytes. - -> _name_to_plan: name -> that module's scatter list - - PLAN once, cached, _build_call_plan - _Chunk one packed pull: its scatters, deduped keys, the byte-exact - pack_layout, which producer serves it, and what to run after - (materialize / quant / free). One per (group, owner class). - _CallPlan all chunks + pre_free. - - RUN per chunk, per sync, _run_chunk_pipeline - _Chunk -> _PendingPull issued, not yet landed: Ray ref, buffer views, - ring slot. [RPC thread] - -> _ProcItem chunk + results + slot. [hand-off to the - background scatter thread] - -The RUN pair stays split on purpose: it is the thread boundary, and only -``targets`` should outlive the get -- carrying the Ray ref and the whole-buffer -blob into the queue would keep both alive for the scatter's lifetime. - -See docs/training/weight_transfer/sharded_rdt.md for the design and the measured -results behind the choices here. -""" - -import time -from dataclasses import dataclass, field -from math import prod -from typing import TYPE_CHECKING, Any, cast - -import torch - -from vllm.config.weight_transfer import WeightTransferConfig -from vllm.distributed.weight_transfer.base import ( - WeightTransferEngine, - WeightTransferInitInfo, - WeightTransferUpdateInfo, -) -from vllm.distributed.weight_transfer.sharded_rdt_common import ( - RdtRouter, - buffer_alloc_bytes, - check_ray_rdt_version, - register_nixl_memory, -) -from vllm.distributed.weight_transfer.sharded_rdt_fake import ( - BakeSink, - FakeRDTTensor, - FetchKey, - _Scatter, -) -from vllm.logger import init_logger - -if TYPE_CHECKING: - from vllm.config import VllmConfig - -logger = init_logger(__name__) - - -@dataclass -class _Chunk: - """One packed pull plus its post-processing (see Data flow). - - A module's copies span chunks when its experts sit in several owner classes, - so ``materialize`` fires on its FIRST chunk and ``quant`` on its LAST -- - materialize-once by construction, not by a runtime counter. ``pack_layout`` - mirrors the producer's rule byte-exactly. - """ - - scatters: "list[_Scatter]" - keys: "list[FetchKey]" - pack_layout: "list[tuple[int, torch.dtype, int, tuple[int, ...]]]" # off,dt,n,shape - pack_bytes: int - materialize: "list[Any]" - quant: "list[Any]" - free: "list[int]" - # Trainer rank serving this chunk, resolved at plan time. One per chunk: - # every name in it shares an owner class, so one producer holds them all. - owner: int - # This chunk's index among the chunks this worker pulls from ``owner``, in - # issue order. It picks the producer's serve slot (``seq % ring_depth``), - # which is what makes slot reuse safe: the pipeline drains pull i before - # issuing i+K, so the slot seq+K takes was last used by seq, whose read is - # finished. Choosing the slot in EXECUTION order instead -- as a per-call - # counter on the producer would -- can hand a slot to a pull whose - # predecessor is still being read, because Ray may start a consumer's - # concurrent produce calls in any order. Static, since the plan is. - seq: int = 0 - - -@dataclass -class _CallPlan: - """The static plan for one sync (see Data flow). Pure, so it is built once - and reused; runtime is then execution only. - - ``pre_free`` = groups with NO chunk on this worker, signaled at sync start - (owners tolerate a signal preceding its publish). With the last-chunk signals - this keeps the completeness invariant consumer-local: every group is signaled - exactly once. - """ - - chunks: "list[_Chunk]" - pre_free: "list[int]" - - -@dataclass -class _PendingPull: - """A dispatched pull whose blocking ``ray.get`` has not run (see Data flow). - - ``targets``/``blob`` must stay strongly referenced until it completes: - ``set_target_for_ref`` stores WEAKREFS, so dropping them silently reroutes the - transfer into a fallback buffer.""" - - ref: "Any" - keys: "list[FetchKey]" - targets: "list[torch.Tensor]" - blob: "torch.Tensor" - slot: int - - -@dataclass -class _ProcItem: - """A landed pull handed to the background scatter thread (see Data flow). - - ``results`` alias the ring buffer ``slot``, held as strong refs so they - outlive the RPC-thread frame until the scatter consumes them. - """ - - chunk: "_Chunk" - results: "dict[FetchKey, torch.Tensor]" - slot: int - - -@dataclass -class ShardedRDTWeightTransferInitInfo(WeightTransferInitInfo): - """Initialization info for the sharded RDT backend.""" - - trainer_actor_names: list[str] = field(default_factory=list) - """Names of all trainer Ray actors exposing the producer method (set via - ``.options(name=...)``), ordered by trainer rank. ``RdtRouter`` picks one of - them per pull, out of the name's owner set. Every actor is bound regardless, - because ``free_group`` fans out to every owner. Must be non-empty; a - single-producer trainer passes a one-element list.""" - - trainer_actor_namespace: str | None = None - """Optional Ray namespace the trainer actor(s) live in.""" - - produce_method_name: str = "rdt_produce_weights_batched" - """Name of the trainer-side producer method. It and the rest of the serve - surface (``free_group``, ``reserve_serve_buffer``) are documented where they - are implemented, on ``_RDTProducerServer`` in ``sharded_rdt_trainer.py``.""" - - names: list[str] = field(default_factory=list) - """The trainer's complete, flat param name list. The bake drives - ``model.load_weights`` over all of them once and keys the plan by source - name.""" - - dtype_names: list[str] = field(default_factory=list) - """Dtype name (e.g. 'bfloat16') for each entry of ``names``.""" - - shapes: list[list[int]] = field(default_factory=list) - """Full HF shape for each entry of ``names``.""" - - group_lens: list[int] = field(default_factory=list) - """Partition of ``names`` into gather groups, in the SAME order the trainers - gather and publish them (group-major; ``sum(group_lens) == len(names)``, and - ``names`` must be ordered to match). Required: the engine pre-builds the whole - static chunk/signal plan from it at init, and fires ``free_group`` at every - owner as each group's last chunk completes.""" - - owner_sets: list[list[int]] = field(default_factory=list) - """The distinct producer sets that occur, each a sorted list of trainer ranks - (indices into ``trainer_actor_names``). Indexed by ``name_owner_class``. - Empty means every producer holds every name.""" - - name_owner_class: list[int] = field(default_factory=list) - """Per-name index into ``owner_sets``, parallel to ``names``: which producers - hold each name. Empty means every producer holds every name. - - This one table expresses every layout the trainer can have — pipeline stages - (the names of these groups have this owner set), expert parallelism (this - expert name has this one-rank owner set), and combinations of the two — so - the engine cuts each group's baked copies into one chunk per distinct class - present and routes each chunk to that class's owner. Derived by the trainer - from ``WeightSource.held_names()``.""" - - num_consumers: int = 0 - """Total consumer count across the fleet, for M:N routing. Authoritative when - > 0; at 0 the engine infers it from ``parallel_config``, which is correct for - the supported serving modes but worth setting explicitly under M:N. Each - worker's distinct index comes from ``_global_worker_index``.""" - - num_rdt_buffers: int = 2 - """Depth of the consumer receive-buffer ring. Must match the - producer's — ``_run_chunk_pipeline``'s slot-safety argument rests on it. 2 = - double buffer: chunk i+1's serve overlaps chunk i's RDMA, and scatter(i-1) - overlaps RDMA(i) in the other slot. Keep depth x chunk_bytes under the - fabric's address-translation reach (~2-3 GB/flow on the reference 8xB200 RoCE - cluster, where K=3 measurably hurt).""" - - buffer_presize_gb: float = 0.0 - """Pre-size each packed receive-buffer slot to this many GiB - (0 = size to the first chunk + coarse 256MB round-up). Set it to cover the - model's largest atomic chunk (e.g. an untied lm_head). Sizing buffers ONCE - matters beyond perf -- see the doc's "Sizing buffers once matters beyond - throughput".""" - - replica_rank: int = 0 - """This inference engine's ordinal in the fleet (0..``num_replicas``-1). - - Multi-engine deployments run several INDEPENDENT inference engines, each with - its own self-contained parallel config, so every engine's - ``_global_worker_index`` restarts at 0 and would collide across engines. The - driver gives each engine a distinct ``replica_rank`` (with identical - ``num_replicas``) so the engine offsets its consumers into a globally distinct - range for the M:N block assignment.Default 0/1 (single - engine) needs no override.""" - - num_replicas: int = 1 - """Number of independent inference engines in the fleet. Default 1 => the - per-replica offset is 0 and consumer identity is exactly - ``_global_worker_index`` (preserves single-engine and single-DP-deployment - behavior). When > 1, ``workers_per_replica = num_consumers // num_replicas`` - and this engine's consumers occupy - ``replica_rank * workers_per_replica + _global_worker_index()``. Assumes a - uniform fleet (every replica has the same worker count). Set alongside - ``replica_rank``, by the driver.""" - - -@dataclass -class ShardedRDTWeightTransferUpdateInfo(WeightTransferUpdateInfo): - """Update info for the sharded RDT backend: intentionally EMPTY. - - The chunk/free plan is a pure function of the baked plan and the driver's - gather-group partition, both fixed for the engine's lifetime, so it is built - once at ``init_transfer_engine`` from ``ShardedRDTWeightTransferInitInfo``'s - ``names`` + ``group_lens``. ONE ``update_weights`` per sync then just re-runs - that plan; there is nothing per-sync to carry. - """ - - -class ShardedRDTWeightTransferEngine( - WeightTransferEngine[ - ShardedRDTWeightTransferInitInfo, - ShardedRDTWeightTransferUpdateInfo, - ] -): - """Pull-based RDT/NIXL backend that transports only the slice each worker - consumes. - - Requires ``distributed_executor_backend="ray"``, ``nixl`` in the shared env, - a named trainer actor exposing a ``@ray.method(tensor_transport="nixl")`` - producer, and weight loaders that stay inside ``SUPPORTED_OPS`` — anything - needing real data (``.to``, ``.item``, arithmetic, bool-mask indexing) - raises during the bake. - - The plan is baked once at ``init_transfer_engine`` into one scatter list - per fully-loaded leaf module, indexed by source name; every - ``update_weights`` replays the modules its gathered names cover. - """ - - init_info_cls = ShardedRDTWeightTransferInitInfo - update_info_cls = ShardedRDTWeightTransferUpdateInfo - # receive_weights pulls synchronously but defers GPU post-processing to a - # background thread so it overlaps the next chunk's pull, so ``update_weights`` - # skips the base's device sync and ``finish_weight_update`` drains the - # deferred work before finalize. - defers_processing = True - # The baked replay plan is a function of one concrete model's parameter - # layout, so a separate draft model cannot reuse it. - supports_draft_weight_update = False - - def __init__( - self, - config: WeightTransferConfig, - vllm_config: "VllmConfig", - device: torch.device, - model: torch.nn.Module, - ) -> None: - super().__init__(config, vllm_config, device, model) - # This engine's data plane is Ray's: the consumer resolves each trainer's - # serve actor with ``ray.get_actor`` and pulls over NIXL through the - # workers' tensor transport, neither of which exists unless the workers - # ARE Ray actors. Caught here, at construction, because the alternative - # is an opaque failure during the first handshake. A single-worker - # deployment defaults to "uni", so "ray" has to be asked for explicitly. - backend = vllm_config.parallel_config.distributed_executor_backend - # A custom Executor subclass is a deliberate override we cannot judge; - # only reject the named backends that are definitely not Ray. - if isinstance(backend, str) and backend != "ray": - raise ValueError( - f"The 'sharded_rdt' weight transfer backend requires " - f"distributed_executor_backend='ray', got {backend!r}. Pass " - f"--distributed-executor-backend ray so the workers are " - f"RayExecutorV2 actors with tensor transport." - ) - # Ownership table, routing rules, and (once bound) every producer's - # handle — pulls go to one owner per chunk, but free_group fans out to - # every owner of the group, so all handles are bound. - self._router: RdtRouter | None = None - # Driver-supplied total consumer count (init_info.num_consumers); 0 => infer - # from parallel_config. See _num_consumers. - self._num_consumers_override: int = 0 - # Source name -> the plan for the module consuming it. Several names of a - # fused module share one entry; replay dedups. A live name absent here - # fails the plan build. - self._name_to_plan: dict[str, list[_Scatter]] = {} - # name -> (dtype_name, shape) for every init name; the bake builds its - # lazies from this. - self._name_meta: dict[str, tuple[str, list[int]]] = {} - # Names whose copy_ fired during the bake. Unbaked names not here never - # move data (e.g. non-local EP experts), so receive_weights skips them. - self._live_names: set[str] = set() - - # ---- Consumer-side pre-registered receive buffers ---------------------- - # One persistent 1-D buffer per dtype per ring slot, STORAGE-registered with - # NIXL once, so every per-slice view hits Ray's registration cache and the - # recv path does zero registration in steady state. See the doc's "Sizing - # buffers once matters beyond throughput" for why they rarely grow. - # - # Ring depth is both the slot count and the in-flight pull count, one - # quantity from ``num_rdt_buffers``. Decoupling them with a spare slot - # measurably slows the sync. - self._ring_depth = 1 - self._dest_buffers: list[dict[torch.dtype, torch.Tensor]] = [{}] - self._slot_read_done: list[Any] = [] # one torch.cuda.Event per slot - # Generation counters guarding the events (see _ensure_proc_worker). - self._slot_queued: list[int] = [] - self._slot_done: list[int] = [] - self._slot_cv: Any = None - self._pull_slot = 0 - # (id(chunk), slot) -> (buffer data_ptr, per-key views). - # Lives on the engine, not on the _Chunk: the plan is static and shared - # across syncs, so a per-slot runtime cache has no business inside it. - self._targets_cache: dict[tuple[int, int], tuple[int, list[torch.Tensor]]] = {} - self._buffer_presize = 0 # bytes; set from init_info - self._pending_frees: list[Any] = [] # free_group signal refs, drained per sync - # The STATIC plan: built once — at init from init_info.group_lens, else - # lazily on the first update_weights — and reused. Non-None means - # update_weights ignores its per-sync names. - self._cached_plan: _CallPlan | None = None - # Completed syncs. The chunk pipeline runs SERIAL during sync 0, when both - # sides still register buffers, and pipelines from sync 1. See the doc. - self._completed_syncs = 0 - - # ---- Background post-processing thread (pull/process pipelining) ------- - # The RPC thread hands pulled buffer views to this worker, which runs - # materialize/scatter/quant/kernel_copy on its own stream while the next - # pull proceeds. drain_pending() joins it before finalize. - self._proc_queue: Any | None = None - self._proc_thread: Any | None = None - self._proc_stream: Any | None = None - # Dedicated quant thread (pass 2): own queue + stream so quant never - # sits between two items' scatters on the scatter thread (measured to - # stall the RPC thread's slot handshake ~0.5-0.75s/iter). - self._quant_queue: Any | None = None - self._quant_thread: Any | None = None - self._quant_stream: Any | None = None - self._proc_error: BaseException | None = None - - def init_transfer_engine(self, init_info: ShardedRDTWeightTransferInitInfo) -> None: - """Configure the ring, bind the producers, bake the replay plan, and - pre-register every NIXL buffer -- in that order, because each step depends - on the previous one. - - The bake drives ``model.load_weights`` and the pre-registration blocks on - RPCs to the producers, so this is a heavyweight one-off; every later - ``update_weights`` is pure replay. - """ - # Read the attribute directly: if a vLLM bump renames it, this guard - # must fail loudly at init rather than silently disappear. - if self.parallel_config.enable_eplb: - raise RuntimeError( - "sharded_rdt does not support EPLB (enable_eplb=true): dynamic " - "expert rearrangement invalidates the baked replay plan, which " - "records each expert's destination slot once at init — after a " - "rearrangement the replay would silently load weights into the " - "wrong expert slots. Disable EPLB or use another weight-transfer " - "backend." - ) - # The worker is often a different install from whatever spawned it, so - # check here too and not only where the actor options are set. - check_ray_rdt_version() - self._configure_ring(init_info) - self._resolve_producers(init_info) - - # A pure dry run: the trainer's gather cache is empty at init, so nothing - # can (or does) get pulled -- we only record how each slice is fetched and - # where it lands, then restore the model. - self._bake(init_info) - self._build_static_plan(init_info) - # Register ALL NIXL memory now, while the fabric is idle: dma-buf GPUDirect - # registration concurrent with in-flight RDMA intermittently fails - # (ibv_reg_mr 'Bad address'), which bites under M:N fan-in. Sizes come from - # the static plan, so this is exact. - self._preregister_at_init() - # Start the background post-processing worker (pull/process pipelining). - self._ensure_proc_worker() - - def _configure_ring(self, init_info: ShardedRDTWeightTransferInitInfo) -> None: - """Ring depth K. - - Must run before ``_ensure_proc_worker`` creates the per-slot events and - counters, and before any buffer is grown (both happen on the first pull). - """ - self._num_consumers_override = int(init_info.num_consumers or 0) - k = max(1, int(init_info.num_rdt_buffers)) - self._ring_depth = k - self._dest_buffers = [{} for _ in range(k)] - self._targets_cache = {} - self._buffer_presize = int(float(init_info.buffer_presize_gb) * (1 << 30)) - logger.info( - "[RDT-RING] active_pulls=%d slots=%d presize=%.2fGiB", - k, - k, - self._buffer_presize / (1 << 30), - ) - - def _resolve_producers(self, init_info: ShardedRDTWeightTransferInitInfo) -> None: - """Work out this worker's consumer identity, build the router, and bind - EVERY producer actor. - - Pull routing is M:N: each chunk goes to ONE producer holding every name - in it (see ``RdtRouter``), so one producer serves a whole pull. All - producers are bound regardless, because the per-group ``free_group`` - signal fans out to every owner of a group, including producers this - worker never pulls from. - """ - try: - import ray - except ImportError as e: - raise RuntimeError( - "Ray is required for the 'sharded_rdt' weight transfer " - "backend. Install Ray and run workers as Ray actors " - "(distributed_executor_backend='ray')." - ) from e - - producer_names = list(init_info.trainer_actor_names) - if not producer_names: - raise RuntimeError( - "Sharded RDT engine requires a trainer producer: set " - "init_info.trainer_actor_names." - ) - - consumer_id = self._resolve_consumer_id(init_info) - name_owner_class = list(init_info.name_owner_class or []) - if name_owner_class and len(name_owner_class) != len(init_info.names): - raise RuntimeError( - f"Sharded RDT engine: {len(name_owner_class)} owner-class entries " - f"for {len(init_info.names)} names." - ) - router = RdtRouter( - len(producer_names), - self._num_consumers(), - list(init_info.owner_sets) or None, - name_owner_class or None, - list(init_info.names), - list(init_info.group_lens), - # Carve the block per deployment so every replica's worker w pulls - # each name from the same producer, which is what lets that producer - # serve all of them from one slot. - workers_per_replica=self._workers_per_replica(init_info), - ) - router.validate() - - # A fresh router per init, never appended to: every owner index is a - # position in these lists, so a rejoining engine's second - # init_transfer_engine must not shift them. - actors: list[Any] = [] - methods: list[Any] = [] - - for chosen_name in producer_names: - try: - actor = ray.get_actor( - chosen_name, - namespace=init_info.trainer_actor_namespace, - ) - except ValueError as e: - raise RuntimeError( - f"Sharded RDT engine could not find trainer actor " - f"{chosen_name!r} (namespace=" - f"{init_info.trainer_actor_namespace!r})." - ) from e - # ``ray.get_actor`` resolves by NAME, in a process that never - # imported the producer's class, and ``enable_tensor_transport`` - # lives in that class's metadata: the creating process passes - # ``meta.enable_tensor_transport``, and Ray infers it from the - # class's ``@ray.method(tensor_transport=...)`` decorators. A - # name-resolved handle therefore always reports False and the - # dispatch guard rejects the pull, even though the trainer set the - # option. Inherent to resolving by name across processes, not a - # version bug; it fails on Ray 2.56.0. - # - # Forcing it skips no validation. Ray's next guard, - # ``actor_has_tensor_transport``, asks the LIVE actor whether it can - # build a NIXL agent and still runs, and ``_spawn_server`` hardcodes - # the actor option, so a misconfigured producer cannot reach here. - actor._ray_enable_tensor_transport = True - actors.append(actor) - methods.append(getattr(actor, init_info.produce_method_name)) - router.bind(actors, methods, consumer_id) - self._router = router - logger.info( - "Sharded RDT engine (consumer %d) bound to all %d producers " - "(batched method %r, %d owner class(es))", - consumer_id, - len(producer_names), - init_info.produce_method_name, - len(init_info.owner_sets) or 1, - ) - - def _workers_per_replica(self, init_info: ShardedRDTWeightTransferInitInfo) -> int: - """Consumers per inference deployment, assuming a uniform fleet. - - Read by this worker's consumer id and by the router's block carve, which - must not disagree, so it is derived once here. - """ - return max(1, self._num_consumers() // max(1, int(init_info.num_replicas or 1))) - - def _resolve_consumer_id(self, init_info: ShardedRDTWeightTransferInitInfo) -> int: - """This worker's DISTINCT index in 0..C-1 across the whole fleet. - - Within one engine that is ``_global_worker_index()``. But a fleet of - INDEPENDENT engines (each with its own parallel config) restarts that - index at 0 per engine, so each engine offsets into its own range using - ``replica_rank``: with a uniform fleet, - ``workers_per_replica = C // num_replicas``. ``num_replicas`` defaults to - 1 (offset 0), preserving single-engine and single-DP-deployment behaviour. - """ - replica_rank = max(0, int(init_info.replica_rank or 0)) - return ( - replica_rank * self._workers_per_replica(init_info) - + self._global_worker_index() - ) - - def _build_static_plan(self, init_info: ShardedRDTWeightTransferInitInfo) -> None: - """Build the chunk/free plan once. It never changes across syncs, so - ``update_weights`` needs no per-sync names.""" - if not init_info.group_lens: - raise ValueError( - "Sharded RDT engine requires init_info.group_lens (the gather-" - "group partition of init_info.names)." - ) - if sum(init_info.group_lens) != len(init_info.names): - raise ValueError( - f"init_info.group_lens sums to {sum(init_info.group_lens)} " - f"but {len(init_info.names)} names were given." - ) - self._cached_plan = self._build_call_plan(init_info.names, init_info.group_lens) - logger.info( - "[RDT-PLAN] pre-built static call plan at init: %d chunks", - len(self._cached_plan.chunks), - ) - - def _preregister_at_init(self) -> None: - """Register every NIXL buffer this worker will use at init, before any - transfer runs, so nothing registers during the sync-0 RDMA churn. - - Both sides are sized from the static plan: receive buffers are - ``ring_depth`` slots at the largest chunk's ``pack_bytes``, and each bound - producer is asked to pre-register a serve ring at the max bytes this - consumer will pull from it. A no-op when this worker has no chunks.""" - plan = self._cached_plan - if plan is None or not plan.chunks: - return - # (a) consumer receive buffers — one per ring slot, at the largest chunk. - max_pack = max(c.pack_bytes for c in plan.chunks) - alloc = buffer_alloc_bytes(max_pack, self._buffer_presize) - for slot in range(self._ring_depth): - buffer = self._dest_buffers[slot].get(torch.uint8) - if buffer is None or buffer.numel() < alloc: - buffer = torch.empty(alloc, dtype=torch.uint8, device=self.device) - register_nixl_memory(buffer) - self._dest_buffers[slot][torch.uint8] = buffer - - # (b) producer serve rings — max bytes this consumer pulls from each - # bound producer, plus the per-producer plan digest a producer sharing - # one ring across deployments checks them against. - assert self._router is not None - serve_bytes = [0] * self._router.num_producers - per_owner_keys: list[list] = [[] for _ in range(self._router.num_producers)] - for c in plan.chunks: - serve_bytes[c.owner] = max(serve_bytes[c.owner], c.pack_bytes) - per_owner_keys[c.owner].append(c.keys) - digests = [_plan_digest(keys) for keys in per_owner_keys] - import ray - - refs = self._router.reserve_serve_buffers(serve_bytes, digests) - if refs: - ray.get(refs) # block until every serve ring is registered - logger.info( - "[RDT-PLAN] pre-registered %d receive slots (%.0f MiB each) + serve " - "rings on %d producer(s) %s", - self._ring_depth, - alloc / (1 << 20), - len(refs), - [nb // (1 << 20) for nb in serve_bytes], - ) - - def _global_worker_index(self) -> int: - """This worker's stable, distinct global index across the inference fleet: - ``data_parallel_index * world_size + rank`` over the TP*PP world. - - ``data_parallel_index``, not ``data_parallel_rank``: vLLM resets the - latter to 0 in a dense worker but keeps the former as the distinct global - DP rank. Same formula as the sibling ``nccl_engine``, so dense-via-TP and - MoE-via-DP+EP both yield distinct 0..C-1.""" - pc = self.parallel_config - return pc.data_parallel_index * pc.world_size + pc.rank # world_size = TP*PP - - def _num_consumers(self) -> int: - """Total inference-worker count. Prefers the driver-supplied - ``init_info.num_consumers`` (authoritative -- the driver knows the whole - fleet); else ``world_size_across_dp``, the same stride - ``_global_worker_index`` indexes with, so the two agree at any pp.""" - if self._num_consumers_override > 0: - return self._num_consumers_override - return self.parallel_config.world_size_across_dp - - def start_weight_update(self) -> None: - """Put the model's params on meta so layerwise reload streams them in - as each layer's slices land. Baked replay uses checkpoint format.""" - from vllm.model_executor.model_loader.reload import ( - initialize_layerwise_reload, - ) - - initialize_layerwise_reload(self.model) - - def finish_weight_update(self) -> None: - """Drain the deferred pull/process pipeline (so every layer is fully - loaded) before finalizing the layerwise reload.""" - from vllm.model_executor.model_loader.reload import ( - finalize_layerwise_reload, - ) - - self.drain_pending() - finalize_layerwise_reload(self.model, self.model_config) - - def update_weights(self, update_info: dict[str, Any]) -> None: - """Receive one update. Unlike the base, does NOT issue a per-update - device sync: post-processing is deferred to background threads and a - sync here would block on them and serialize the pull/process pipeline. - Completion is guaranteed by ``drain_pending`` in - ``finish_weight_update``.""" - self.receive_weights(self.parse_update_info(update_info)) - - def receive_weights( - self, - update_info: ShardedRDTWeightTransferUpdateInfo, - ) -> None: - """Pull + replay the baked leaf modules the sync covers. - - The chunk/free plan is STATIC across syncs — a pure function of the baked - plan and the driver's group partition — so it was built once at init and - every sync just re-runs the pipeline over its self-describing chunks, with - no per-sync bookkeeping and an empty ``update_info``. - - Assumes each baked module's source names fall within one gather group, - which the per-layer / pre / post partition guarantees (a leaf module's - sources all live in one decoder layer). A module that did span groups - would be planned once per group and could pull a name whose group the - pipeline already freed, which parks the pull until the producer's stall - watchdog fires. - """ - del update_info # the plan is static; nothing arrives per sync - if self._router is None: - raise RuntimeError( - "Sharded RDT engine not initialized. Call init_transfer_engine() first." - ) - # Surface any error the background thread hit on a prior item promptly. - self._raise_proc_error() - if self._cached_plan is None: - raise RuntimeError( - "Sharded RDT engine has no call plan: init_info.group_lens must " - "be supplied at init_transfer_engine()." - ) - self._run_chunk_pipeline(self._cached_plan) - - def _build_fake_weights( - self, - names: list[str], - sink: "BakeSink", - device: torch.device, - ) -> list[tuple[str, torch.Tensor]]: - """Zero-storage lazies for ``names``, dtype/shape from the init metadata, - all feeding the bake's recording sink.""" - return [ - ( - name, - FakeRDTTensor( - name=name, - shape=torch.Size(self._name_meta[name][1]), - dtype=_dtype_from_name(self._name_meta[name][0]), - device=device, - sink=sink, - ), - ) - for name in names - ] - - def _issue_pull(self, chunk: "_Chunk", slot: int) -> "_PendingPull": - """Reserve ``slot``, lay the targets out in its buffer, dispatch the - produce RPC and point the transfer at the buffer — WITHOUT the blocking - ``ray.get`` (that is ``_complete_pull``). The chunked pipeline issues - chunk i+1 before completing chunk i, so the producer serves the next - chunk while the in-flight RDMA streams. - - Slot-reuse guard, both stages required: a generation wait (the CUDA event - binds only to its LAST record, so synchronizing before the background - thread recorded this item's event passes silently — observed as - nondeterministic weight corruption), then the event synchronize. It must - precede ``set_target_for_ref``, not just the get: the transfer may start - any time after the metadata push. See the doc's "slot generation - handshake". - """ - from ray.experimental import set_target_for_ref - - if self._slot_read_done: - with self._slot_cv: - while self._slot_done[slot] < self._slot_queued[slot]: - self._slot_cv.wait(timeout=1.0) - self._slot_read_done[slot].synchronize() - - # Every slice is byte-packed into ONE uint8 buffer, 16B-aligned in - # keys order, under one NIXL descriptor. ``chunk.pack_layout`` mirrors the - # producer's rule byte-exactly; ``targets`` carves the dtype views back out. - keys = chunk.keys - cur = chunk.pack_bytes - buffers = self._dest_buffers[slot] - buffer = buffers.get(torch.uint8) - if buffer is None or buffer.numel() < cur: - # Size ONCE with headroom: Ray's desc cache is keyed by data_ptr and - # outlives its tensor, so a regrowth can false-hit a recycled pointer - # and skip registering the new extent -> NIXL_ERR_NOT_FOUND. - alloc = buffer_alloc_bytes(cur, self._buffer_presize) - buffer = torch.empty(alloc, dtype=torch.uint8, device=self.device) - register_nixl_memory(buffer) - buffers[torch.uint8] = buffer - targets = self._pull_targets(chunk, slot, buffer) - # One produce RPC serves the whole packed chunk, since the group's owner - # holds every slice. The buffer view must stay strongly referenced through - # the get: set_target_for_ref stores WEAKREFS, and a dropped target - # reroutes the transfer into a fallback buffer. - blob = buffer[:cur] - assert self._router is not None - ref = self._router.pull(chunk.owner, keys, chunk.seq) - set_target_for_ref(ref, [blob]) - return _PendingPull( - ref=ref, - keys=keys, - targets=targets, - blob=blob, - slot=slot, - ) - - def _pull_targets( - self, chunk: "_Chunk", slot: int, buffer: torch.Tensor - ) -> "list[torch.Tensor]": - """Per-key dtype views into ``slot``'s buffer. - - The packed layout is static, so the views are built once per (chunk, slot) - rather than once per pull -- rebuilding them cost ~1150 Python ops per pull - at 235B. Keyed on the buffer pointer as well, so a regrow invalidates - instead of handing back views into a freed buffer. - """ - cached = self._targets_cache.get((id(chunk), slot)) - if cached is not None and cached[0] == buffer.data_ptr(): - return cached[1] - targets = [ - buffer[off : off + n * dt.itemsize].view(dt).reshape(shape) - for off, dt, n, shape in chunk.pack_layout - ] - self._targets_cache[(id(chunk), slot)] = (buffer.data_ptr(), targets) - return targets - - def _complete_pull(self, pending: "_PendingPull") -> "dict[FetchKey, torch.Tensor]": - """Blocking half of a pull: the NIXL read lands during this ``ray.get``.""" - import ray - - ray.get(pending.ref) - return dict(zip(pending.keys, pending.targets)) - - # ---------------- Bake (dry run, at init) / replay ---------------- - - def _bake(self, init_info: ShardedRDTWeightTransferInitInfo) -> None: - """Bake the replay plan once, as a self-driven meta dry run. - - Puts the params on meta, then drives ``model.load_weights`` over - ``init_info.names`` through the model's ORIGINAL loaders (the stamps - bypass ``online_process_loader``, so ``_layerwise_process`` is never in - the path). Nothing materializes or pulls; the fake's ``copy_`` records - the source op chain and the meta destination's geometry. Afterwards one - scatter list per fully-loaded leaf module (copied numel == loadable - size) is indexed by source name; a partial or unrecordable module fails - the plan build. The model is restored. - - This leans on layerwise internals a public API should expose first-class: - a currently-loading hook instead of monkeypatched stamps, a dry-run mode - instead of bypassing ``online_process_loader``, and an - ``abort_layerwise_reload`` instead of ``_restore_after_dry_run``. - """ - from vllm.model_executor.model_loader.reload.layerwise import ( - initialize_layerwise_reload, - ) - from vllm.model_executor.model_loader.reload.utils import get_layer_size - - names, dtype_names, shapes = ( - init_info.names, - init_info.dtype_names, - init_info.shapes, - ) - self._name_meta = {n: (d, s) for n, d, s in zip(names, dtype_names, shapes)} - if not names: - return - - model = self.model - recorder = BakeSink() - - _t0 = time.perf_counter() - with torch.device(self.device): - # Meta-restore params + save kernel tensors (we bypass the loader - # wrapping it installs, below). - initialize_layerwise_reload(model) - # Stamp the *original* loaders (bypassing online_process_loader), so - # the single load pass runs the loaders on meta and records via the - # fake's copy_ — with no inline _layerwise_process, no deferral. - self._install_recording_stamps(model, recorder) - model.load_weights(self._build_fake_weights(names, recorder, self.device)) - # Keep only fully-loaded modules (copied numel >= loadable size, the - # test online_process_loader uses): a partial module leaves unwritten - # regions that finalize would init, so baking it scatters garbage. - for module, recorded in recorder.copies_by_layer.items(): - if not recorded or any(c is None for c in recorded): - continue # unrecordable copy_ -> slow path - # Guard above guarantees every entry is a real _Scatter. - copies = cast("list[_Scatter]", recorded) - copied = sum(prod(c.shape) for c in copies) - if copied < get_layer_size(module): - continue # partial -> slow path - for c in copies: - # Every name of the module shares ONE list, so identity - # dedups them back to one module at plan time. - self._name_to_plan[c.src[0]] = copies - self._restore_after_dry_run(model) - - # Names whose copy_ fired during the bake. Names not in here no-op for - # this worker (e.g. foreign-EP experts) and are skipped; live names that - # did not bake fail the plan build. - self._live_names = set(recorder.copied_names) - - n_groups = len({id(g) for g in self._name_to_plan.values()}) - logger.info( - "Sharded RDT dry-run baked %d/%d names into %d leaf modules " - "(%d live) in %.3fs", - len(self._name_to_plan), - len(names), - n_groups, - len(self._live_names), - time.perf_counter() - _t0, - ) - - def _install_recording_stamps( - self, model: torch.nn.Module, recorder: "BakeSink" - ) -> None: - """Wrap each loadable param's ``weight_loader`` to stamp - ``recorder.current = (leaf_module, param_name)`` before delegating to the - original loader, so the fake's ``copy_`` can attribute each recorded copy. - ``functools.wraps`` keeps the loader's real signature (so vLLM's - ``_layerwise_process`` ``param`` redirect still works if a stamp leaks), - and ``_rdt_stamp_inner`` tags it so ``_restore_after_dry_run`` can unwrap it. - """ - import functools - - from vllm.model_executor.model_loader.reload.layerwise import ( - _get_original_loader, - ) - from vllm.model_executor.model_loader.reload.utils import get_layer_tensors - from vllm.model_executor.model_loader.weight_utils import default_weight_loader - - def _make_stamp(layer, name, inner, added=False): - @functools.wraps(inner) # keep ``inner``'s signature (incl. ``param``) - def stamp(*args, **kwargs): - recorder.current = (layer, name) - try: - return inner(*args, **kwargs) - finally: - recorder.current = None - - # Tag so _restore_after_dry_run can detect and unwrap leaked stamps, - # and so a second bake doesn't double-wrap. - stamp._rdt_stamp_inner = inner # type: ignore[attr-defined] - stamp._rdt_stamp_added = added # type: ignore[attr-defined] - return stamp - - for module in model.modules(): - for name, tensor in get_layer_tensors(module).items(): - if getattr(tensor, "weight_loader", None) is None: - # A param with NO loader (e.g. GLM's router bias, a plain - # nn.Parameter) is still loaded by the model's load_weights - # through the getattr(param, "weight_loader", - # default_weight_loader) fallback — unstamped, its bake copy - # would be unattributable, failing the module's coverage - # gate and the plan build. Stamp the same default loader - # the fallback would pick; the restore deletes it again. - tensor.weight_loader = _make_stamp( - module, name, default_weight_loader, added=True - ) - continue - # Bypass online_process_loader: stamp the *original* loader. - original = _get_original_loader(tensor) - tensor.weight_loader = _make_stamp(module, name, original) - - def _restore_after_dry_run(self, model: torch.nn.Module) -> None: - """Restore each layerwise layer's saved kernel tensors without pulling - (a real ``finalize_layerwise_reload`` would materialize/load) and reset - its info. Also unwrap any recording ``stamp`` left on the params, since a - leaked stamp would sit under the next sync's ``online_process_loader`` and - silently break ``_layerwise_process``'s ``param`` redirect. - """ - from vllm.model_executor.model_loader.reload.layerwise import ( - LAYERWISE_INFO, - _place_kernel_tensors, - ) - from vllm.model_executor.model_loader.reload.utils import get_layer_tensors - - for layer in model.modules(): - info = LAYERWISE_INFO.get(layer) - if info is not None and info.can_load(): - if info.kernel_tensors is not None: - _place_kernel_tensors(layer, info) - info.reset() - # Unwrap any recording stamps left on the (now-restored) params so they - # never leak into a later update_weights. ``_rdt_stamp_inner`` is set by - # ``_install_recording_stamps``; unwrap repeatedly in case of nesting. - for module in model.modules(): - for _name, tensor in get_layer_tensors(module).items(): - loader = getattr(tensor, "weight_loader", None) - added = False - while loader is not None and hasattr(loader, "_rdt_stamp_inner"): - added = added or getattr(loader, "_rdt_stamp_added", False) - loader = loader._rdt_stamp_inner - tensor.weight_loader = loader - if added: - # The stamp was ATTACHED to a param that had no loader - # (see _install_recording_stamps); leave none behind. - del tensor.weight_loader - if hasattr(model, "_original_do_torchao_reload"): - model._do_torchao_reload = model._original_do_torchao_reload - - # ---------------- Background post-processing (pull/process pipeline) ------- - - def _ensure_proc_worker(self) -> None: - """Lazily create the per-slot events, the background CUDA stream, the work - queue, and the single processing thread. Idempotent.""" - if self._proc_thread is not None: - return - import queue - import threading - - self._slot_read_done = [torch.cuda.Event() for _ in range(self._ring_depth)] - # Generation handshake for the events above: the RPC thread counts items - # queued per slot, this thread counts records, and a pull may synchronize() - # only once done[slot] has caught up with queued[slot]. Without it the - # synchronize can bind to a stale record and pass, letting the next RDMA - # overwrite a slot under a pending scatter. See the doc's "The slot - # generation handshake". - self._slot_queued = [0] * self._ring_depth - self._slot_done = [0] * self._ring_depth - self._slot_cv = threading.Condition() - self._proc_stream = torch.cuda.Stream(device=self.device) - self._proc_queue = queue.Queue() - self._proc_error = None - t = threading.Thread( - target=self._proc_worker_loop, name="rdt-postprocess", daemon=True - ) - self._proc_thread = t - t.start() - self._quant_stream = torch.cuda.Stream(device=self.device) - self._quant_queue = queue.Queue() - qt = threading.Thread( - target=self._quant_worker_loop, name="rdt-quant", daemon=True - ) - self._quant_thread = qt - qt.start() - - def _proc_worker_loop(self) -> None: - """Single persistent thread: run each queued item's process phase on the - background stream. Exits on the ``None`` sentinel (shutdown). An item that - raises is recorded in ``_proc_error`` and re-raised on the RPC thread / - at drain, so a failed sync fails loudly rather than corrupting silently. - """ - torch.cuda.set_device(self.device) - q = self._proc_queue - assert q is not None - while True: - item = q.get() - try: - if item is None: - return - # _process_item publishes the slot generation internally (in a - # finally around its scatter pass), so an error here still - # unblocks the RPC thread's generation wait. - self._process_item(item) - except BaseException as e: # noqa: BLE001 - surfaced on the RPC thread - self._proc_error = e - logger.exception("RDT background post-processing failed") - finally: - q.task_done() - - def _mark_slot_done(self, slot: int) -> None: - """Publish that a queued item's read-done event has been recorded (or the - item failed) so a pull waiting to reuse ``slot`` can proceed to its - CUDA-event synchronize.""" - with self._slot_cv: - self._slot_done[slot] += 1 - self._slot_cv.notify_all() - - def _raise_proc_error(self) -> None: - """Re-raise (once) any error captured by the background thread.""" - if self._proc_error is not None: - err = self._proc_error - self._proc_error = None - raise RuntimeError("RDT background post-processing failed") from err - - def drain_pending(self) -> None: - """Block until the background thread has processed every queued item and - its stream work is complete, then re-raise any error it hit. Called from - the worker's ``finish_weight_update`` before ``finalize_layerwise_reload`` - so every baked layer is fully loaded (and ``info.reset()``-ed) first.""" - if self._proc_queue is not None: - self._proc_queue.join() # every put() item task_done()'d - # The scatter thread feeds the quant thread, so join it SECOND (all - # completed-group batches have been put by now), then sync both streams - # so finalize sees fully-materialized, quanted, reset layers. - if self._quant_queue is not None: - self._quant_queue.join() - if self._proc_stream is not None: - self._proc_stream.synchronize() - if self._quant_stream is not None: - self._quant_stream.synchronize() - self._raise_proc_error() - # Ensure every fired free_group signal has EXECUTED on - # the producer before the sync ends: ``begin_sync`` resets the producer's - # per-group signal counts, so a signal landing after the next sync - # started would credit a group it does not belong to, over-crediting the - # gather-lookahead backpressure (extra gather resident -> trainer OOM - # risk). - if self._pending_frees: - import ray - - try: - ray.get(self._pending_frees) - finally: - self._pending_frees.clear() - # One sync iteration fully drained; buffers/registrations are at (or - # nearer) high-water — the chunk pipeline may issue ahead from now on. - self._completed_syncs += 1 - - def _dispatch_item(self, item: "_ProcItem") -> None: - """Hand one chunk item to the background scatter thread. - - Counts the item against its slot BEFORE dispatch: the next pull into - that slot must wait until the background thread has processed (and - RECORDED the read-done event for) every item ever queued on it. - """ - with self._slot_cv: - self._slot_queued[item.slot] += 1 - assert self._proc_queue is not None - self._proc_queue.put(item) - - def _chunk_module_scatters( - self, modules: "list[list[_Scatter]]" - ) -> "list[tuple[int, list[_Scatter]]]": - """Cut the modules' copies into one chunk per distinct owner class - present, ascending by class index, as ``(class_idx, scatters)`` pairs. - - A chunk is one packed pull, so every name in it must share a producer — - which is exactly what an owner class is. The cut is a pure function of - the bake and the ownership table. Copy order within a chunk is bake - order, and a module's copies may span chunks (materialize/quant fire on - its first/last chunk; see ``_build_call_plan``).""" - assert self._router is not None - by_class: dict[int, list[_Scatter]] = {} - for copies in modules: - for c in copies: - by_class.setdefault(self._router.class_of(c.src[0]), []).append(c) - return [(ci, by_class[ci]) for ci in sorted(by_class)] - - def _build_call_plan(self, names: list[str], group_lens: list[int]) -> "_CallPlan": - """Build the STATIC plan for one whole-sync call. - - Pure — no pulls, no engine state touched — so the result is cached and - reused every sync. Three passes: - 1. Split ``names`` into gather groups, one chunk per owner class - present in this worker's baked copies, recording each group's last - chunk for its ``free_group`` signal (or ``pre_free`` when this worker - pulls nothing from it). Every group is signaled exactly once by - construction. The stream has no per-group call boundaries, so group - L+1's first chunk issues while L's still stream. - 2. Per leaf module, find its FIRST and LAST chunk — materialize on the - first, quant/kernel/reset on the last, correct by construction - instead of by runtime counters. - 3. Assemble ``_Chunk``s: dedup keys and precompute the packed layout. - """ - router = self._router - assert router is not None, "init_transfer_engine() must run before planning" - - # --- pass 1: gather groups -> per-owner-class scatter chunks ----------- - raw_chunks: list[list[_Scatter]] = [] - free_at: dict[int, list[int]] = {} # chunk idx -> groups to signal after it - pre_free: list[int] = [] # groups with no chunk here (signal at start) - unbaked: list[str] = [] - pos = 0 - for gi, glen in enumerate(group_lens): - gnames = names[pos : pos + glen] - pos += glen - modules: list[list[_Scatter]] = [] - seen: set[int] = set() - for n in gnames: - mod = self._name_to_plan.get(n) - if mod is None: - if n in self._live_names: - unbaked.append(n) - elif id(mod) not in seen: - seen.add(id(mod)) - modules.append(mod) - if not modules: - # Nothing to pull for this group on this worker; its owners - # still published it, so signal it done at sync start (they - # tolerate a signal that arrives before the publish). - pre_free.append(gi) - continue - for _ci, scatters in self._chunk_module_scatters(modules): - raw_chunks.append(scatters) - free_at.setdefault(len(raw_chunks) - 1, []).append(gi) - if unbaked: - # A live name with no baked plan cannot be loaded: there is no - # fallback, and its pull would target groups the pipeline has - # already freed. Fail here, where the names are known. - raise RuntimeError( - f"Sharded RDT: {len(unbaked)} live name(s) did not bake, " - f"first: {unbaked[:3]}. Fix the loader so these names bake " - "(fully load their module in one pass), or exclude them from " - "the synced set." - ) - - # --- pass 2: per-module first/last chunk -> materialize/quant ---------- - first_at: dict[int, int] = {} - last_at: dict[int, int] = {} - layer_by_id: dict[int, Any] = {} - for ci, scatters in enumerate(raw_chunks): - for sc in scatters: - lid = id(sc.layer) - layer_by_id[lid] = sc.layer - first_at.setdefault(lid, ci) - last_at[lid] = ci - materialize_at: dict[int, list[Any]] = {} - quant_at: dict[int, list[Any]] = {} - for lid, ci in first_at.items(): - materialize_at.setdefault(ci, []).append(layer_by_id[lid]) - for lid, ci in last_at.items(): - quant_at.setdefault(ci, []).append(layer_by_id[lid]) - - # --- pass 3: assemble _Chunks (dedup keys + precompute pack layout) ---- - from collections import defaultdict - - chunks: list[_Chunk] = [] - per_owner_seq: dict[int, int] = defaultdict(int) - for ci, scatters in enumerate(raw_chunks): - keys: list[FetchKey] = [] - kmeta: dict[FetchKey, tuple[torch.dtype, tuple[int, ...]]] = {} - for sc in scatters: - if sc.src not in kmeta: - kmeta[sc.src] = (sc.dtype, sc.shape) - keys.append(sc.src) - pack_layout: list[tuple[int, torch.dtype, int, tuple[int, ...]]] = [] - cur = 0 - for k in keys: - dt, shape = kmeta[k] - numel = prod(shape) or 1 # type: ignore[arg-type] - off = (cur + 15) & ~15 - pack_layout.append((off, dt, numel, shape)) - cur = off + numel * dt.itemsize - owner = router.producer_for(router.consumer_id, scatters[0].src[0]) - chunks.append( - _Chunk( - seq=per_owner_seq[owner], - scatters=scatters, - keys=keys, - pack_layout=pack_layout, - pack_bytes=cur, - materialize=materialize_at.get(ci, []), - quant=quant_at.get(ci, []), - free=free_at.get(ci, []), - # Every name of a chunk shares an owner class and a group, - # so any of them resolves the same producer. - owner=owner, - ) - ) - per_owner_seq[owner] += 1 - return _CallPlan(chunks=chunks, pre_free=pre_free) - - def _signal_group_done(self, group_idx: int) -> None: - """Fire-and-forget ``free_group`` signal at EVERY owner of the group. - - The per-group barrier: each owner counts one signal per live consumer - and frees the group (releasing its lookahead credit) on the last one, so - every owner must hear from every consumer — including owners this worker - pulls nothing from. Refs are held and drained in ``drain_pending`` so - every signal has EXECUTED before the sync ends: ``begin_sync`` resets - the counters, and a straggler landing in the next sync would credit a - group it does not belong to.""" - assert self._router is not None - self._pending_frees.extend(self._router.free_group(group_idx)) - - def _run_chunk_pipeline(self, plan: "_CallPlan") -> None: - """Pipelined chunk pulls over the ring of receive slots. - - Issues up to ``ring_depth`` produce RPCs ahead of the blocking gets, so - while chunk i's RDMA streams the producer serves i+1 into its own ring - slot and the background thread scatters i-1 out of another. Reads stay - serialized on the shared NIC — the bandwidth floor, not a loss. - - Slot safety rests on two arguments spelled out in the doc: the producer's - ring is no shallower than this one and drain-before-issue orders its - reuse; the consumer's slots are held by ``_issue_pull``'s generation - handshake. - """ - from collections import deque - - inflight: deque[tuple[_PendingPull, _Chunk]] = deque() - - # Gather groups with no chunk on this worker (plan.pre_free): signal - # them done before the pipeline — their owners still published them, and - # a signal that arrives before its publish is counted (fire-and-forget). - for gi in plan.pre_free: - self._signal_group_done(gi) - - def drain_one() -> None: - pending, chunk = inflight.popleft() - results = self._complete_pull(pending) - self._dispatch_item( - _ProcItem(chunk=chunk, results=results, slot=pending.slot) - ) - # Each gather group whose LAST chunk this is: its read is done, so - # this consumer is finished with the group -> signal every owner - # (fire-and-forget, off the critical path). - for gi in chunk.free: - self._signal_group_done(gi) - - for chunk in plan.chunks: - if not chunk.keys: - # _chunk_module_scatters never emits empty chunks; keep the - # signals safe anyway if this ever changes. - for gi in chunk.free: - self._signal_group_done(gi) - continue - # Drain BEFORE issue once the ring is full: frees this chunk's slot - # (generation-wise) and guarantees the producer-slot invariant above. - # Sync 0 runs SERIAL (max 1 in flight): both sides still grow and - # register buffers, and a producer registration mid-flight churns its - # agent-metadata version under an in-flight pull (see __init__). - depth = 1 if self._completed_syncs == 0 else self._ring_depth - if len(inflight) >= depth: - drain_one() - slot = self._pull_slot - self._pull_slot = (slot + 1) % self._ring_depth - inflight.append((self._issue_pull(chunk, slot), chunk)) - while inflight: - drain_one() - - def _process_item(self, item: "_ProcItem") -> None: - """Scatter-thread half: materialize this chunk's first-seen modules, - scatter its slices on the process stream, publish the slot, then hand the - modules it COMPLETES to the quant thread. - - Mirrors ``_layerwise_process`` minus the loader replay. Once every scatter - reading ``item.slot`` is enqueued, records the slot's read-done event so - the RPC thread can block on it before overwriting the slot. - """ - from vllm.model_executor.model_loader.reload.layerwise import ( - LAYERWISE_INFO, - ) - from vllm.model_executor.model_loader.reload.meta import materialize_layer - - results = item.results - chunk = item.chunk - with ( - torch.cuda.device(self.device), - torch.cuda.stream(self._proc_stream), - torch.device(self.device), - ): - # PASS 1 — slot readers: materialize the modules whose FIRST scatter - # is in this chunk (empty HF params, once per module by construction) - # then scatter this chunk's copies. The scatter copies are the ONLY - # reads of the receive buffer; quant and kernel-copy operate on the - # scattered params. Releasing the slot right after the scatters lets - # the NEXT chunk's RDMA overwrite the buffer while quant still runs. - try: - for layer in chunk.materialize: - info = LAYERWISE_INFO.get(layer) - if info is None or not info.can_load(): - raise RuntimeError( - f"Baked replay: layer {type(layer).__name__} " - "was not set up for reload this sync " - "(start_weight_update must run before " - "update_weights)." - ) - materialize_layer(layer, info) - for sc in chunk.scatters: - param = getattr(sc.layer, sc.param_name) - # Through .data, exactly like default_weight_loader: params - # that keep requires_grad=True (plain nn.Parameters like - # GLM's router bias) reject an in-place copy through an - # autograd view of the leaf. - dst = param.data.as_strided(sc.shape, sc.stride, sc.offset) - with torch._C.DisableTorchFunctionSubclass(): - dst.copy_(results[sc.src]) - # All reads of this slot's buffer are now enqueued on the process - # stream; record + publish so the RPC thread can reuse the slot. - self._slot_read_done[item.slot].record(self._proc_stream) - finally: - # Publish even on error so the RPC thread's generation wait - # unblocks (the failure itself surfaces via _raise_proc_error). - self._mark_slot_done(item.slot) - - # PASS 2 — param readers only: quant / kernel-copy / reset for the - # modules whose LAST scatter is in this chunk. Handed to the - # DEDICATED quant thread (own CUDA stream, event-chained after this - # chunk's scatters) so it never delays this thread's next pass-1. - # Running it in order here stalls the RPC thread's slot handshake by - # ~0.5-0.75s/iter, since every group's quant delays the next publish. - if chunk.quant: - ready = torch.cuda.Event() - ready.record(self._proc_stream) - if self._quant_queue is None: - self._run_quant(chunk.quant, ready) - else: - self._quant_queue.put((chunk.quant, ready)) - - def _run_quant(self, layers: "list[Any]", ready: "torch.cuda.Event") -> None: - """Quant/kernel-copy/reset the given COMPLETED leaf modules, exactly as - _layerwise_process. Runs on the quant thread's own stream, ordered after - the modules' scatters via ``ready``; touches only the scattered params - (never a receive slot), so it can overlap subsequent chunks' RDMA and - scatters. ``info.reset()`` is what makes finalize skip the layer — - drain_pending joins the quant queue before finalize runs.""" - from vllm.model_executor.layers.quantization.base_config import ( - QuantizeMethodBase, - ) - from vllm.model_executor.model_loader.reload.layerwise import ( - LAYERWISE_INFO, - _copy_and_restore_kernel_tensors, - ) - - stream = self._quant_stream or self._proc_stream - assert stream is not None # created in _ensure_proc_worker before use - with ( - torch.cuda.device(self.device), - torch.cuda.stream(stream), - torch.device(self.device), - ): - stream.wait_event(ready) - for layer in layers: - info = LAYERWISE_INFO.get(layer) - assert info is not None # completed leaf module is set up for reload - quant_method = getattr(layer, "quant_method", None) - if isinstance(quant_method, QuantizeMethodBase): - if hasattr(layer, "_already_called_process_weights_after_loading"): - delattr(layer, "_already_called_process_weights_after_loading") - quant_method.process_weights_after_loading(layer) - # Copy into persistent kernel storage (preserves cudagraph refs). - if info.kernel_tensors is not None: - _copy_and_restore_kernel_tensors(layer, info) - # Reset so finalize_layerwise_reload skips this (loaded) layer. - info.reset() - - def _quant_worker_loop(self) -> None: - """Dedicated quant thread: drains (completed_modules, scatter-done event) - batches. Errors surface via _proc_error like the scatter thread's.""" - torch.cuda.set_device(self.device) - q = self._quant_queue - assert q is not None - while True: - batch = q.get() - try: - if batch is None: - return - self._run_quant(*batch) - except BaseException as e: # noqa: BLE001 - surfaced on the RPC thread - self._proc_error = e - logger.exception("RDT quant thread failed") - finally: - q.task_done() - - def shutdown(self) -> None: - # Stop the background post-processing thread (drain, then sentinel + join) - # before dropping the state it touches. - if self._proc_thread is not None: - try: - self.drain_pending() - except Exception: - logger.exception("RDT drain during shutdown failed") - assert self._proc_queue is not None - self._proc_queue.put(None) # sentinel - self._proc_thread.join(timeout=30) - self._proc_thread = None - self._proc_queue = None - self._proc_stream = None - if self._quant_thread is not None: - assert self._quant_queue is not None - self._quant_queue.put(None) # sentinel - self._quant_thread.join(timeout=30) - self._quant_thread = None - self._quant_queue = None - self._quant_stream = None - self._slot_read_done = [] - self._router = None - # Drop strong references to baked modules so the model can be freed. - self._name_to_plan.clear() - self._name_meta.clear() - self._live_names.clear() - # Drop the cached plan (holds _Scatter refs to the baked layers). - self._cached_plan = None - # Release the receive buffers (their NIXL registration is pinned for the - # process lifetime; freeing the tensors just drops our strong refs). - self._dest_buffers = [{} for _ in range(self._ring_depth)] - - -def _plan_digest(keys_per_chunk: list) -> str: - """Digest of the chunks one consumer pulls from one producer, in pull order. - - Two consumers a producer serves out of ONE shared serve ring must agree on - this, since sharing rests on their plans being identical; the producer - compares it at init. Over the whole (name, op-chain) list rather than the - names, because the chains decide the bytes each pull returns. - """ - import hashlib - - h = hashlib.sha256() - for keys in keys_per_chunk: - h.update(repr(keys).encode()) - h.update(b"\n") - return h.hexdigest()[:32] - - -def _dtype_from_name(name: str) -> torch.dtype: - """Resolve a string like 'bfloat16' to torch.bfloat16.""" - dtype = getattr(torch, name, None) - if not isinstance(dtype, torch.dtype): - raise ValueError(f"Unknown torch dtype name: {name!r}") - return dtype diff --git a/vllm/distributed/weight_transfer/sharded_rdt_fake.py b/vllm/distributed/weight_transfer/sharded_rdt_fake.py deleted file mode 100644 index 5e3613e9d5..0000000000 --- a/vllm/distributed/weight_transfer/sharded_rdt_fake.py +++ /dev/null @@ -1,300 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Op-chain recording for the sharded-RDT backend. - -The consumer asks the trainer for the exact slice a worker consumes, described as -an op chain replayed on the trainer's live tensor. ``FakeRDTTensor`` builds the -chain by intercepting the model's own weight loaders; ``copy_`` is its data sink, -``BakeSink`` — the dry run records how each slice would be fetched and where it -lands. - -Chains are built against meta tensors, so nothing here touches Ray or a GPU. -""" - -from collections import defaultdict -from collections.abc import Callable -from dataclasses import dataclass, field -from typing import Any - -import torch - -from vllm.distributed.weight_transfer.sharded_rdt_common import SUPPORTED_OPS - -# ("op_name", positional_args, sorted_kwargs_items). Entries must be hashable so -# the chain can serve as a FetchKey. -OpSpec = tuple[str, tuple[Any, ...], tuple[tuple[str, Any], ...]] -OpChain = tuple[OpSpec, ...] -FetchKey = tuple[str, OpChain] - - -def _freeze_kwargs(kwargs: dict[str, Any]) -> tuple[tuple[str, Any], ...]: - """Sort kwargs into a tuple of items for hashable storage in OpSpec.""" - return tuple(sorted(kwargs.items())) - - -class _UnsupportedFakeOp(NotImplementedError): - """Raised when a weight loader does something to a FakeRDTTensor that cannot - be expressed as a slice request. - - Surfaced as NotImplementedError so callers can distinguish "this backend - can't handle this loader" from genuine bugs. - """ - - -@dataclass -class _Scatter: - """One recorded scatter: pull ``src``, copy it into ``layer``'s - ``param_name`` at the recorded strided region. The bake's output unit; see - the engine module's Data flow. - - Self-contained so replay needs no lookups. Destination geometry is read off - the meta view and rebuilt each sync as ``as_strided(shape, stride, offset)`` - -- ``layer`` is resolved to a param at replay time, never baked, since every - sync re-materializes fresh tensors. - - ``shape`` and ``dtype`` also size the slice ON THE WIRE, so ``dtype`` is the - PRODUCED dtype (the fake's after its op chain), not the source name's: - ``view(dtype)`` is allowlisted, and taking the source's would size the slice - with the wrong itemsize and carve the packed blob differently on the two - sides. - """ - - layer: Any - param_name: str - src: FetchKey - offset: int - shape: tuple[int, ...] - stride: tuple[int, ...] - dtype: torch.dtype - - -def _meta_copy_(dest: torch.Tensor, src: "FakeRDTTensor") -> torch.Tensor: - """Fire ``dest.copy_`` from a zero-storage meta source of ``src``'s geometry. - - Moves no data but still counts against the layer's loaded numel, which drives - ``_layerwise_process``; skipping it would leave the layer looking unloaded - forever. - - A non-meta ``dest`` is one the layerwise reload never moved there (the - ``SKIP_LOAD_TENSORS`` set, e.g. GLM's router bias). Torch forbids - ``real.copy_(meta)``, and skipping the count is consistent -- ``get_layer_size`` - excludes the same set. The caller still RECORDS the copy, so the replay writes - the real param. - """ - if not dest.is_meta: - return dest - meta_src = torch.empty(src.shape, dtype=src.dtype, device="meta") - with torch._C.DisableTorchFunctionSubclass(): - return dest.copy_(meta_src) - - -@dataclass -class BakeSink: - """``copy_`` sink for the dry-run bake: record how each slice would be fetched - and where it would land, and move nothing. - - ``_install_recording_stamps`` sets ``current = (leaf_module, param_name)`` - around each loader so ``accept_copy`` can attribute the copy. An unstamped - ``copy_`` cannot be attributed and stays unrecorded, so its module fails the - coverage gate and takes the plain load. ``copies_by_layer`` is keyed by module - object, so iterating it yields each leaf module once. - """ - - copies_by_layer: "dict[Any, list[_Scatter | None]]" = field( - default_factory=lambda: defaultdict(list) - ) - current: "tuple[Any, str] | None" = None - # Source names whose ``copy_`` fired during the bake. Names not here never - # moved data (e.g. experts owned by another EP rank) and are skipped; names - # here but unbaked fail the plan build. - copied_names: "set[str]" = field(default_factory=set) - - def accept_copy(self, dest: torch.Tensor, src: "FakeRDTTensor") -> torch.Tensor: - self.copied_names.add(src._name) - if self.current is not None: - layer, param_name = self.current - if dest.numel() != src.numel(): - # The packed layout sizes each slice from the DESTINATION shape, - # while the producer packs what the replayed chain produces. A - # broadcasting copy_ makes those two lengths differ, and neither - # side can notice: the consumer's buffer view is exactly - # prod(dest.shape) elements, so it reshapes cleanly over bytes - # the producer laid out at different offsets. Refuse at bake. - raise _UnsupportedFakeOp( - f"FakeRDTTensor: broadcasting copy_ into {param_name!r} of " - f"{type(layer).__name__} ({src.numel()} elements from " - f"{src._name!r} into {dest.numel()}). The sharded RDT " - "backend sends exactly the slice the loader asks for, so " - "the source and destination must have equal numel." - ) - self.copies_by_layer[layer].append( - _Scatter( - layer=layer, - param_name=param_name, - src=src._key(), - offset=dest.storage_offset(), - shape=tuple(dest.shape), - stride=tuple(dest.stride()), - dtype=src.dtype, - ) - ) - return _meta_copy_(dest, src) - - -class FakeRDTTensor(torch.Tensor): - """Zero-storage tensor that records how to fetch a weight slice. - - ``_make_wrapper_subclass`` gives it shape/dtype/device without storage. Every - op in ``SUPPORTED_OPS`` returns a child with the spec appended; ``copy_`` - delegates to the installed sink. Anything else reaches ``__torch_dispatch__`` - and raises ``_UnsupportedFakeOp``, so failures are loud rather than silently - fetching the wrong bytes. - """ - - # Declared at class scope so mypy can infer attribute types on - # instances built via ``_make_wrapper_subclass`` (where ``__new__`` - # returns a tensor it can't annotate as ``self``). - _name: str - _ops: OpChain - # Handles this fake's copy_. ``None`` only on bare construction. - _sink: "BakeSink | None" - - @staticmethod - def __new__( - cls, - name: str, - shape: torch.Size, - dtype: torch.dtype, - device: torch.device, - ops: OpChain = (), - sink: "BakeSink | None" = None, - ) -> "FakeRDTTensor": - t = torch.Tensor._make_wrapper_subclass( - cls, - shape, - dtype=dtype, - device=device, - requires_grad=False, - ) - t._name = name - t._ops = tuple(ops) - t._sink = sink - return t - - def _key(self) -> FetchKey: - return (self._name, self._ops) - - def _make_child( - self, - new_shape: torch.Size, - new_dtype: torch.dtype, - *new_ops: OpSpec, - ) -> "FakeRDTTensor": - """Append one or more ops to the chain and return a fresh child. - - Variadic so multi-return ops (e.g. chunk) can append both the base - op and an indexing op in a single call. - """ - return FakeRDTTensor( - name=self._name, - shape=new_shape, - dtype=new_dtype, - device=self.device, - ops=self._ops + new_ops, - sink=self._sink, - ) - - def _meta(self) -> torch.Tensor: - """A zero-storage meta tensor of this fake's shape/dtype, so PyTorch - itself computes post-op geometry rather than us reimplementing it.""" - return torch.empty(self.shape, dtype=self.dtype, device="meta") - - @classmethod - def __torch_function__( - cls, - func, - types, - args=(), - kwargs=None, - ): - kwargs = kwargs or {} - - # Checked before the allowlist: copy_ must not be recorded into the chain. - if func is torch.Tensor.copy_: - dest = args[0] - src = args[1] if len(args) > 1 else kwargs.get("src") - if isinstance(src, cls) and src._sink is not None: - return src._sink.accept_copy(dest, src) - - # Allowlisted slice/view/shape ops: append to chain and return child. - op_name = SUPPORTED_OPS.get(func) - if op_name is not None: - self_ = args[0] - if isinstance(self_, cls): - rest = tuple(args[1:]) - return cls._intercept(self_, func, op_name, rest, kwargs) - - # Metadata reads are answered by the wrapper subclass and stop here; ops - # that need data fall through to __torch_dispatch__, which raises. - with torch._C.DisableTorchFunctionSubclass(): - return func(*args, **kwargs) - - @classmethod - def _intercept( - cls, - self_: "FakeRDTTensor", - func: Callable, - op_name: str, - args: tuple, - kwargs: dict, - ): - """Append the op and return a child, or a tuple of children for - multi-return ops. Each child's geometry comes from running the op on a - meta tensor.""" - meta = self_._meta() - with torch._C.DisableTorchFunctionSubclass(): - meta_result = func(meta, *args, **kwargs) - - base_op: OpSpec = (op_name, tuple(args), _freeze_kwargs(kwargs)) - - # Single-tensor result: one child carrying base_op alone. - if isinstance(meta_result, torch.Tensor): - return self_._make_child(meta_result.shape, meta_result.dtype, base_op) - - # Multi-return: one child per output, each carrying base_op plus a - # trailing __getitem__(i) so the replay can index the result. - if isinstance(meta_result, tuple | list): - return tuple( - self_._make_child( - m.shape, - m.dtype, - base_op, - ("__getitem__", (i,), ()), - ) - for i, m in enumerate(meta_result) - ) - - # Non-tensor result (e.g. .item() snuck into the allowlist). - raise _UnsupportedFakeOp( - f"FakeRDTTensor: {op_name!r} returned a non-tensor " - f"({type(meta_result).__name__}); cannot defer." - ) - - @classmethod - def __torch_dispatch__(cls, func, types, args=(), kwargs=None): - kwargs = kwargs or {} - - # A fake at the aten level means an unsupported loader op. Name the op and - # the chain; materializing instead would mask a correctness bug. - for arg in (*args, *kwargs.values()): - if isinstance(arg, cls): - raise _UnsupportedFakeOp( - f"FakeRDTTensor: unsupported op {func} reached " - f"__torch_dispatch__ on fake {arg._name!r} " - f"(chain={arg._ops}). Supported ops are: " - f"{sorted(SUPPORTED_OPS.values())}, plus copy_. " - "Loaders that need .to(), .float(), .item(), arithmetic, " - "bool-mask indexing, or .data access are not supported by " - "the sharded RDT backend." - ) - return func(*args, **kwargs) diff --git a/vllm/distributed/weight_transfer/sharded_rdt_trainer.py b/vllm/distributed/weight_transfer/sharded_rdt_trainer.py deleted file mode 100644 index 51ea401ef9..0000000000 --- a/vllm/distributed/weight_transfer/sharded_rdt_trainer.py +++ /dev/null @@ -1,1520 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Trainer-side engine for the sharded-RDT (pull-based NIXL) backend. - -RDT is pull-based, so unlike NCCL this engine broadcasts nothing. It owns a -per-rank producer server (an internal Ray actor exposing the NIXL serve surface -the worker engine dials by name), and on each ``send_weights`` gathers this -rank's weights group-by-group from the ``WeightSource``, shares each group into -the server over CUDA IPC, and — on the sender — drives the inference-side -start/update/finish handshake, whose single empty ``update_weights`` unblocks the -workers to pull. - -All serve-side state lives on the server actor, so trainer processes need no -mixin, no named actors and no special actor options. - -See docs/training/weight_transfer/sharded_rdt.md for the publish -> serve -> -free_group -> release lifecycle and the ownership model. -""" - -import contextlib -import threading -import time -import uuid -from collections.abc import Callable, Collection -from concurrent.futures import ThreadPoolExecutor -from dataclasses import asdict, dataclass, field -from typing import Any, ClassVar, Self - -import ray -import torch -from torch.multiprocessing.reductions import ( - StorageWeakRef, - rebuild_cuda_tensor, - reduce_tensor, -) - -from vllm.distributed.weight_transfer.base import ( - ParamMeta, - TrainerInitInfo, - TrainerWeightTransferEngine, - VLLMWeightSyncClient, - WeightSource, - layerwise_groups, -) -from vllm.distributed.weight_transfer.sharded_rdt_common import ( - ALLOWED_OPS, - buffer_alloc_bytes, - check_ray_rdt_version, - register_nixl_memory, -) -from vllm.logger import init_logger - -logger = init_logger(__name__) - -# Gathered-but-unfreed groups the gather loop may run ahead by before it stops -# gathering. A gathered group is published immediately (publish_group never -# blocks), so at most ``lookahead + 1`` groups are resident on the trainer — the -# memory bound larger models size against, pinned by the tests. -# -# At 1, group N+1 is gathered and serveable while the consumers pull N, so the -# boundary's free-barrier latency hides behind live pulls. Raise it only if one -# group's gather is slower than its pulls. -DEFAULT_GATHER_LOOKAHEAD = 1 - -# Seconds with no publish, produce or free before the -# producer declares the sync dead. A consumer that dies mid-sync never signals -# ``free_group``, so the group is never released and the waits below block -# forever — which stops this rank iterating its WeightSource, a collective, and -# wedges every other trainer rank with no exception anywhere. This converts that -# into one real error. -# -# A liveness backstop, not a latency target: nominal gaps are sub-second, so this -# is ~100x margin. It rides the init info because the producer is a Ray actor and -# does not inherit the trainer's environment. -DEFAULT_STALL_TIMEOUT_S = 300.0 -# How often a blocked waiter re-checks the progress stamp. -_STALL_POLL_S = 5.0 - -# Tensors whose STORAGE is at most this are packed into a ring slot and exported -# as one CUDA-IPC handle; larger storages are exported directly. Batching the -# small ones is what keeps sync time steady -- a per-tensor handle for each of -# ~37k names dominated the variance -- and `_pack_group_for_export` is also the -# only publish path that orders the pack against the sidecar's IPC read, so the -# packing is not optional. -_EXPORT_RING_MAX_BYTES = 64 << 20 - -# The actor method the worker engine dials for the NIXL pull; fixed by contract. -PRODUCE_METHOD_NAME = "rdt_produce_weights_batched" - - -# Cross-deployment serve-slot sharing. -# -# Consumers whose ids differ by a multiple of ``workers_per_replica`` are the -# same worker of different inference deployments, so they share a baked plan and -# pull byte-identical chunks. A serve ring each would cost one full ring per -# deployment of the producer's GPU and repack the same bytes once per deployment. -# -# NIXL reads are one-sided, so R readers can read ONE registered slot; what the -# producer cannot see is when a reader has FINISHED. The release edge is -# therefore the consumer's ISSUE order, sent as ``seq``: the pipeline drains pull -# i before issuing i+K, so slot ``seq % K`` was last packed for ``seq - K``, -# whose read is over. A counter on THIS side would follow execution order -# instead, which is wrong -- Ray may start a consumer's K concurrent produce -# calls in any order, so a call that executes K-before another can be a pull -# still being read, and its slot gets repacked underneath the reader. Silent, -# showing up only as a logprob drift. -# -# Hence one rendezvous per generation, keyed by ``seq``: the group's live sharers -# meet there, the LAST to arrive packs, and all return that one blob. It carries -# the release edge too, since ``seq`` is packed only once every live sharer has -# arrived, so each has drained ``seq - K``. Mismatched plans would rendezvous on -# differing bytes, so they are rejected at init by ``reserve_serve_buffer``'s -# ``plan_digest``. A sharer that dies mid-sync stalls its group, exactly as a -# dead consumer already stalls the per-group free barrier; the next -# ``begin_sync`` narrows the live set. -# -# LIMITATION: the rendezvous synchronizes deployments that a pull-based transfer -# would otherwise leave independent -- per chunk every sharer waits for the -# slowest, so skew between their DP pause/resume brackets lands in every -# deployment's sync time. Nothing about P2P requires this. It is the price of -# serving out of GPU memory, which is too scarce for a ring per consumer, so -# slots must be reused, and reuse needs a release edge. -# -# TODO: explore staging packed chunks in HOST memory. Host buffers are cheap -# enough for a ring per consumer, which would drop the rendezvous, the -# cross-replica coupling and the slot-reuse hazard together. The cost is a D2H -# bounce on the serve path, which has been the bottleneck before, so it needs -# measuring rather than assuming. - - -@dataclass -class _SharedPack: - """One generation of one slot-sharing group: the rendezvous state for a - single chunk, identified by the consumers' issue index.""" - - group: int - slot: int - """``seq % ring_depth``. Fixed at creation, so slot reuse follows the - consumers' issue order rather than this side's execution order.""" - arrived: set[int] = field(default_factory=set) - blob: Any = None - packing: bool = False - done: bool = False - error: BaseException | None = None - - -@dataclass -class ShardedRDTTrainerInitInfo(TrainerInitInfo): - """Trainer init info for the sharded-RDT backend. - - Identical on every rank except ``rank`` (rank 0 is the sender). Carries only - the must-agree wire params; the sender forwards them verbatim onto the - worker-side init info so the two cannot drift. Server-actor names are - generated per rank and all-gathered by the engine, not supplied here. - """ - - backend: ClassVar[str] = "sharded_rdt" - - num_consumers: int - """Total inference-worker (consumer) count across the whole fleet - (DP*TP*PP*PCP), for the M:N block assignment / free ref-count.""" - workers_per_replica: int = 0 - """Consumers per inference DEPLOYMENT (``num_consumers // num_replicas``). - - Fixes the slot-sharing groups: consumers whose ids differ by a multiple of - this are the same worker of different deployments, so they bake identical - plans and pull byte-identical chunks, and ONE registered serve slot can - serve all of them. 0 disables sharing, and so does the single-deployment - value ``num_consumers``, which makes every group a singleton and the serve - path identical to the unshared one. - """ - trainer_actor_namespace: str | None = None - """Ray namespace the engine spawns its serve actors in. The inference - workers (which run in their own EngineCore subprocess with its own - ``ray.init``) resolve those actors by name, so this must be the namespace - they can see. Forwarded to the worker-side init info.""" - num_rdt_buffers: int = 2 - """Serve/receive ring depth K (must match the worker).""" - buffer_presize_gb: float = 0.0 - """Serve-buffer pre-size floor in GiB (avoids NIXL desc-cache churn).""" - gather_lookahead: int = DEFAULT_GATHER_LOOKAHEAD - """Gathered-but-unfreed groups the gather loop may run ahead by. Bounds - trainer-resident memory at ``gather_lookahead + 1`` groups.""" - stall_timeout_s: float = DEFAULT_STALL_TIMEOUT_S - """Seconds of no publish/serve/free progress before the producer fails the - sync (see ``DEFAULT_STALL_TIMEOUT_S``).""" - - -class _RDTProducerServer: - """Per-rank NIXL serve surface: an internal Ray actor sharing the trainer - rank's GPU over CUDA IPC. Holds the gather cache, per-consumer serve rings, - the per-group free barrier and the packed serve. The engine feeds it with - ``publish_group``; workers pull with ``rdt_produce_weights_batched`` and - signal with ``free_group``. - - A plain class — the engine wraps it with ``ray.remote(...)`` at spawn so the - actor options live in one place. - """ - - def __init__( - self, - *, - num_rdt_buffers: int, - buffer_presize_gb: float, - gather_lookahead: int, - served_names: list[str] | None = None, - stall_timeout_s: float = DEFAULT_STALL_TIMEOUT_S, - workers_per_replica: int = 0, - ) -> None: - import gc - - self._device_index = torch.accelerator.current_device_index() - # name -> rebuilt CUDA-IPC tensor (or view); guarded by _cache_cond. - self._cache: dict[str, torch.Tensor] = {} - self._cache_cond = threading.Condition() - self._gather_error: BaseException | None = None - - # Monotonic stamp of the last forward step of the - # publish -> serve -> free loop. The waits below poll it instead of - # blocking forever, so a dead consumer fails the sync with a real error. - self._stall_timeout = float(stall_timeout_s) - self._last_progress = time.monotonic() - - # Names this producer publishes, so a misrouted pull fails loudly - # instead of blocking forever in the cache wait. None = serve anything. - self._served_names = set(served_names) if served_names is not None else None - - # Every live consumer signals free_group(gi) at every - # owner, once per sync; the group frees when the count reaches - # begin_sync's live total — one uniform target, no routed per-producer - # ones. Signals may precede the publish, which completes them. - self._live_count = 1 - self._free_counts: dict[int, int] = {} - # gi -> the names this producer published for it (what release drops). - self._group_names: dict[int, list[str]] = {} - - # Published-but-unfreed groups, plus freed ones not - # yet handed back. The memory gate lives in the ENGINE's gather loop; this - # side only accounts — free_group moves a group from _inflight_groups to - # _freed_pending, which the engine collects via wait_freed / end_sync to - # drop its storage refs. _lookahead is engine-enforced, unused here. - self._lookahead = max(0, gather_lookahead) - self._inflight_groups: list[int] = [] - self._freed_pending: list[int] = [] - - # Ring of packed serve buffers, one ring per SLOT-SHARING GROUP (one - # consumer per group unless several deployments share); the slot within - # it is the consumer's ``seq % nring``. - self._nring = max(1, num_rdt_buffers) - self._serve_rings: dict[int, list[torch.Tensor | None]] = {} - self._serve_lock = threading.Lock() - # registerMem on a shared NIXL agent is not concurrency-safe; serialize. - self._reg_lock = threading.Lock() - self._buffer_presize = int(buffer_presize_gb * (1 << 30)) - self._serve_device = torch.device("cuda", self._device_index) - - # Slot-sharing state, all guarded by _cache_cond, so the rendezvous - # waits ride the stall watchdog like every other wait here. - # ``_share_width`` = consumers per deployment; 0 means every group is a - # singleton and this degenerates to the unshared serve path. - self._share_width = max(0, int(workers_per_replica)) - self._live_ids: set[int] | None = None - self._sharing_active = False - self._sharers: dict[int, frozenset] = {} # group -> live members - self._gens: dict[int, dict] = {} # group -> {seq: _SharedPack} - # group -> {cid: plan digest}. Init-time state, not cleared per sync: it - # is what proves the sharers of a group pull the same chunks. - self._plan_digests: dict[int, dict[int, str]] = {} - - # (sharing group, ring idx, packed layout) -> - # (buffer data_ptr, destination views). Keyed on the buffer pointer too, so - # a ring regrow invalidates rather than writing into a freed buffer. See - # the serve path for why the layout, not the spec names, is the key. - self._pack_dsts: dict[tuple, tuple[int, list[torch.Tensor]]] = {} - # [RDT-EXPORT-RING] handle -> rebuilt base. The trainer reuses buffers - # across groups, so the same handle arrives on many publishes; without - # this we reopen it every group. Cleared per sync so no IPC mapping into - # trainer memory outlives the sync that made it. - self._base_cache: dict[tuple, torch.Tensor] = {} - - # Freeze the static post-init object graph so gen-2 GC never stops the - # world mid-serve, which shows up as serve stragglers. - gc.collect() - gc.freeze() - - def ping(self) -> int: - return self._device_index - - # ---------------- stall watchdog ---------------- - - def _note_progress_locked(self) -> None: - """Record that the credit loop moved. Caller holds ``_cache_cond``.""" - self._last_progress = time.monotonic() - - def _wait_for(self, blocked: Callable[[], bool], what: str) -> None: - """``_cache_cond.wait()`` with a liveness bound. - - Waits while ``blocked()`` holds, returning early on a gather error. If - nothing on this producer progresses for ``_stall_timeout``, self-fires - ``set_gather_error``, which every waiter here already checks — so the rank - unwinds through one path and the driver gets a real exception. - - The progress stamp is global to the producer, not per-waiter: a merely - slow waiter is kept alive by its peers' progress. Caller holds - ``_cache_cond``. - """ - while blocked(): - if self._gather_error is not None: - return - self._cache_cond.wait(_STALL_POLL_S) - if not blocked() or self._gather_error is not None: - return - stalled = time.monotonic() - self._last_progress - if stalled >= self._stall_timeout: - msg = ( - f"RDT stall: no progress for {stalled:.0f}s while waiting for " - f"{what} (timeout {self._stall_timeout:.0f}s). A consumer most " - f"likely died mid-sync: {len(self._inflight_groups)} group(s) " - f"published and unfreed." - ) - logger.error("[rdt-stall] %s", msg) - # Same channel a gather failure uses: wakes every waiter here, and - # each raises rather than returning a half-served result. - self._gather_error = RuntimeError(msg) - self._cache_cond.notify_all() - return - - def warmup_nixl(self) -> None: - """Create this server's NIXL agent now, while the rank's GPU is quiet. - - Called at spawn, before the server-name all-gather, so no rank can be - spinning in a collective. Creating the agent lazily instead deadlocks on - EFA-class fabrics (see "warmup_nixl breaks a startup deadlock" in the - doc). The warmup buffer stays registered so the agent's CUDA-HMEM path - stays initialized. - """ - self._nixl_warmup_buf = torch.zeros(1 << 20, dtype=torch.uint8, device="cuda") - with self._reg_lock: - register_nixl_memory(self._nixl_warmup_buf) - - # ---------------- engine-facing (per sync) ---------------- - - def begin_sync( - self, live_count: int, live_consumer_ids: list | None = None - ) -> None: - """Reset per-sync free/backpressure state and set this sync's barrier - target. - - ``live_count`` is how many consumers take part in THIS sync. Required, no - default: a forgotten argument silently targeting 1 would free groups after - the FIRST signal while others still pull — use-after-free, not an error. - The driver awaits the previous sync's finish before the next begins, so - nothing is in flight; a straggler signal would otherwise credit the wrong - sync, which is why the consumer drains its signals before finishing. - - ``live_consumer_ids`` is that same live set enumerated, which is what - sizes each slot-sharing group's rendezvous; a count cannot, since a group - is a specific set of ids. ``None`` leaves every group a singleton. - - The packed-destination cache deliberately survives: the layout repeats - every sync. So does ``_plan_digests``, which is init-time state. - """ - with self._cache_cond: - self._gather_error = None - self._live_count = max(1, int(live_count)) - self._live_ids = ( - set(int(c) for c in live_consumer_ids) - if live_consumer_ids is not None - else None - ) - # Sharing is on only when the geometry shows several deployments; - # one deployment takes the singleton path. - self._sharing_active = ( - self._share_width > 0 - and self._live_ids is not None - and len(self._live_ids) > self._share_width - ) - self._sharers.clear() - self._gens.clear() - self._inflight_groups.clear() - self._freed_pending.clear() - self._free_counts.clear() - self._group_names.clear() - self._base_cache.clear() - self._note_progress_locked() - - def _release_group_locked(self, group_idx: int) -> None: - """Drop a freed group's cache entries and queue it for the engine - (whose gather-credit gate blocks in ``wait_freed`` on exactly this). - - Shared by the last ``free_group`` and by ``publish_group`` completing an - early-signaled group. Caller must hold ``_cache_cond``. - """ - for name in self._group_names.pop(group_idx, ()): - self._cache.pop(name, None) - self._free_counts.pop(group_idx, None) - if group_idx in self._inflight_groups: - self._inflight_groups.remove(group_idx) - self._freed_pending.append(group_idx) - - def publish_group(self, group_idx: int, entries: tuple) -> None: - """Rebuild one gather group's CUDA-IPC tensors into the serve cache. - - NEVER blocks: a gathered group is serveable immediately. The memory bound - lives in the engine's gather loop, which stops GATHERING (not publishing) - past ``gather_lookahead`` unfreed groups. - - ``entries`` is ``(storages, views)``: one CUDA-IPC export per storage, - plus per-name ``(sid, dtype_name, shape, stride, storage_offset)`` rebuilt - here as ``as_strided`` views. - - Signals can arrive BEFORE their publish — a consumer pulling nothing of a - group signals it as its plan starts — so a group whose barrier is already - satisfied is released here. Freed groups reach the engine only through - ``wait_freed`` / ``end_sync``: a freed notice riding an unharvested async - publish result would wedge the loop. - """ - storages, views = entries - bases: dict[int, torch.Tensor] = {} - for sid, reduce_args in storages.items(): - cached = self._base_cache.get(reduce_args) - if cached is not None: - bases[sid] = cached - continue - list_args = list(reduce_args) - # Index 6 of reduce_tensor's args is the exporter's device index; - # rebuild on this server's device (same physical GPU as the rank). - list_args[6] = self._device_index - bases[sid] = self._base_cache[reduce_args] = rebuild_cuda_tensor(*list_args) - rebuilt: dict[str, torch.Tensor] = {} - for name, (sid, dtype_name, shape, stride, storage_offset) in views.items(): - typed = bases[sid].view(getattr(torch, dtype_name)) - rebuilt[name] = torch.as_strided(typed, shape, stride, storage_offset) - del bases - - with self._cache_cond: - self._cache.update(rebuilt) - self._inflight_groups.append(group_idx) - self._group_names[group_idx] = list(rebuilt) - if self._free_counts.get(group_idx, 0) >= self._live_count: - self._release_group_locked(group_idx) - self._note_progress_locked() - self._cache_cond.notify_all() - - def wait_freed(self) -> list[int]: - """Block until at least one published group has been freed; return and - clear the freed backlog. The engine's gather-credit gate calls this once - its loop is ``gather_lookahead`` groups ahead, so this wait is where the - trainer paces to the consumers' pull rate. - - Raises rather than returning empty when the sync errors or stalls: the - engine is blocked here inside its gather loop, and an empty return would - spin it straight back in. - """ - with self._cache_cond: - self._wait_for(lambda: not self._freed_pending, "a freed-group credit") - if not self._freed_pending: - raise RuntimeError( - "gather errored while waiting for a freed-group credit: " - f"{self._gather_error!r}" - ) - freed = self._freed_pending - self._freed_pending = [] - return freed - - def end_sync(self) -> list[int]: - """Block until every published group has been freed by its consumers; - return the remaining freed keys so the engine drops its last refs.""" - with self._cache_cond: - self._wait_for( - lambda: bool(self._inflight_groups), - "every published group to be freed", - ) - freed = self._freed_pending - self._freed_pending = [] - self._base_cache.clear() - return freed - - def set_gather_error(self, message: str) -> None: - """Record a trainer-side gather failure so blocked serves / publishes - stop waiting and surface it.""" - with self._cache_cond: - self._gather_error = RuntimeError(message) - self._cache_cond.notify_all() - - # ---------------- consumer-facing (called by name over Ray) ---------------- - - def free_group(self, group_idx: int) -> None: - """Consumer back-edge: one consumer is done with group ``group_idx``, - either because its last chunk landed or because it had nothing to pull. - - The per-group barrier counts one signal per live consumer against the - ``begin_sync`` count; every consumer signals every owner, so the target is - the same integer everywhere. The last signal drops the cache entries and - queues the group as gather credit for ``wait_freed``. A signal arriving - before its publish is only counted — ``publish_group`` completes it. - """ - gi = int(group_idx) - with self._cache_cond: - count = self._free_counts.get(gi, 0) + 1 - self._free_counts[gi] = count - if gi in self._group_names and count >= self._live_count: - self._release_group_locked(gi) - self._note_progress_locked() - self._cache_cond.notify_all() - - def _new_serve_buffer(self, nbytes: int) -> torch.Tensor: - """Allocate + NIXL-register one serve slot: the single allocation seam, - so registration cannot be skipped on either of the two paths that make - buffers (the init-time reservation and the serve-path backstop).""" - t = torch.empty(nbytes, dtype=torch.uint8, device=self._serve_device) - with self._reg_lock: - register_nixl_memory(t) - return t - - def reserve_serve_buffer( - self, consumer_id: int, nbytes: int, plan_digest: str | None = None - ) -> None: - """Pre-allocate + NIXL-register this consumer's serve ring before any - pull, while the fabric is idle (avoids registration races during the - sync-0 RDMA churn under M:N fan-in). Idempotent; grows only if needed. - - The ring is keyed by SLOT-SHARING GROUP, so the sharers of one group - reserve ONE ring between them. They all call this, same group and same - size, so the whole body runs under ``_serve_lock``, or two concurrent - first-callers would each allocate a ring and one would be dropped while - still registered. - - ``plan_digest`` is the caller's ordered chunk plan for THIS producer. - Sharers must agree on it, and this is where a disagreement is caught - rather than at serve time, where it is a rendezvous nobody completes. - - Raises: - RuntimeError: two consumers of one sharing group disagree on the - chunks they pull from this producer. - - """ - sg = self._share_group(consumer_id) - if plan_digest is not None: - with self._cache_cond: - seen = self._plan_digests.setdefault(sg, {}) - clash = next( - ((c, d) for c, d in seen.items() if d != plan_digest), None - ) - if clash is None: - seen[consumer_id] = plan_digest - if clash is not None: - raise RuntimeError( - f"consumers {clash[0]} and {consumer_id} are in slot-sharing " - f"group {sg} but do not pull the same chunks from this " - f"producer ({clash[1]} != {plan_digest}). Sharing a serve " - f"slot requires the deployments to be identical; check that " - f"workers_per_replica matches the inference fleet's " - f"geometry." - ) - # This is where the producer's NIXL serve memory is committed: one ring - # of K slots per SHARING GROUP that pulls from this rank -- with a single - # deployment, one ring per consumer, since every group is a singleton. - # Routing only sends a consumer here for names this rank owns, so the - # count is the fan-in, not the fleet. - # - # Every slot is sized by the LARGEST chunk that consumer pulls from us, - # rounded up to a 256 MB multiple, so the whole ring pays for the worst - # chunk in the plan while most chunks are far smaller. - # - # TODO: that over-allocates. Sizing slots per chunk instead of per plan, - # or capping chunk bytes so no chunk is an outlier, would shrink it; so - # would dropping the 256 MB round-up, which is slack once a buffer never - # regrows. Row-splitting the vocab matrix shrinks the outlier itself. - # The host-staging TODO above would remove these rings altogether. - alloc = buffer_alloc_bytes(nbytes, self._buffer_presize) - with self._serve_lock: - rings = self._serve_rings.setdefault(sg, [None] * self._nring) - for i in range(self._nring): - slot = rings[i] - if slot is None or slot.numel() < alloc: - rings[i] = self._new_serve_buffer(alloc) - - def _share_group(self, consumer_id: int) -> int: - """The slot-sharing group ``consumer_id`` belongs to: its index within - its own deployment, since that is what fixes the plan. Without a width - the group is the consumer itself and nothing is shared.""" - return consumer_id % self._share_width if self._share_width > 0 else consumer_id - - def _sharers_of(self, sg: int) -> frozenset: - """The LIVE consumers of group ``sg``: the rendezvous width. Derived from - ``begin_sync``'s live set, so a degraded sync narrows the barrier instead - of waiting forever on a dead deployment. Caller holds ``_cache_cond``.""" - cached = self._sharers.get(sg) - if cached is None: - if not self._sharing_active or self._live_ids is None: - cached = frozenset((sg,)) - else: - cached = frozenset( - c for c in self._live_ids if c % self._share_width == sg - ) or frozenset((sg,)) - self._sharers[sg] = cached - return cached - - def _join_share_group(self, consumer_id: int, seq: int) -> tuple: - """Register this call's arrival in its group's rendezvous for ``seq`` and - return ``(generation, is_packer)``. - - Exactly one caller per generation gets ``is_packer=True``, whichever - completes the arrival set, and it is the only one that touches the GPU. - The slot is ``seq % nring``: fixed by the consumers' issue order, so it - needs no release accounting on this side. - """ - sg = self._share_group(consumer_id) - with self._cache_cond: - gens = self._gens.setdefault(sg, {}) - gen = gens.get(seq) - if gen is None: - gen = gens[seq] = _SharedPack(group=sg, slot=seq % self._nring) - gen.arrived.add(consumer_id) - if gen.packing or gen.done or not gen.arrived >= self._sharers_of(sg): - return gen, False - gen.packing = True - return gen, True - - def _await_shared_pack(self, gen: _SharedPack) -> torch.Tensor: - """Block until this generation's packer published its blob, and return - it. A failed pack is re-raised here, so every sharer of a bad pack fails - instead of reading a half-written slot.""" - with self._cache_cond: - self._wait_for( - lambda: not gen.done and gen.error is None, - "a co-replica's shared serve pack", - ) - if gen.error is not None: - raise RuntimeError( - f"the sharer packing this chunk failed: {gen.error!r}" - ) from gen.error - if not gen.done: - raise RuntimeError( - f"gather errored while awaiting a shared pack: " - f"{self._gather_error!r}" - ) - return gen.blob - - def _serve_slot(self, sg: int, idx: int, need: int) -> torch.Tensor: - """Group ``sg``'s ring slot ``idx``, grown if this chunk outgrew the - reservation. Growing registers memory while the fabric is busy, the - hazard ``reserve_serve_buffer`` exists to avoid, so it is a backstop: the - reservation is sized from the same static plan as this pack.""" - with self._serve_lock: - buffer = self._serve_rings.setdefault(sg, [None] * self._nring)[idx] - if buffer is not None and buffer.numel() >= need: - return buffer - buffer = self._new_serve_buffer(buffer_alloc_bytes(need, self._buffer_presize)) - with self._serve_lock: - self._serve_rings.setdefault(sg, [None] * self._nring)[idx] = buffer - return buffer - - def _fail_shared_pack(self, gen: _SharedPack, exc: BaseException) -> None: - """Publish a pack failure so waiting sharers raise instead of hanging on - a generation that will never complete.""" - with self._cache_cond: - gen.error = exc - self._cache_cond.notify_all() - - @ray.method(tensor_transport="nixl") - def rdt_produce_weights_batched( - self, specs: list, consumer_id: int = 0, seq: int = -1 - ): - """Serve one batched slice request over NIXL. - - Waits until the specs' names are cached, then rendezvouses with the - other live sharers of this consumer's slot-sharing group. The last to - arrive replays each spec's op chain (pure views into cached tensors, - guarded by ALLOWED_OPS) and byte-packs the slices 16B-aligned into the - group's ring slot ``seq % nring``, mirroring the consumer's identical - layout; every sharer then returns that one packed blob, so R deployments - cost one pack and one slot instead of R. With a single deployment every - group is a singleton, so the arriving call is always its own packer and - this is the unshared serve path exactly. - - ``seq`` is the caller's index in its pull stream to this producer, which - is what fixes the slot. Callers that want a single slice pass one spec - and read the blob back with that slice's dtype/shape. - - Raises: - ValueError: ``seq`` was not supplied. - - """ - needed = sorted({n for n, _ in specs}) - if self._served_names is not None: - unserved = [n for n in needed if n not in self._served_names] - if unserved: - # Without this the cache wait below would block forever: this - # producer never gathers these names, so they can never arrive. - raise RuntimeError( - f"pull routed to the wrong producer: {unserved[:3]} " - f"({len(unserved)} names) are not served here" - ) - with self._cache_cond: - self._wait_for( - lambda: not all(n in self._cache for n in needed), - f"{len(needed)} name(s) to be published", - ) - if not all(n in self._cache for n in needed): - # _wait_for only gives up on a gather error (its own stall - # included), so the names are never coming. - raise RuntimeError( - f"gather errored before {needed}: {self._gather_error!r}" - ) - - if seq < 0: - # The slot is derived from it, so a caller that does not send one - # cannot be served safely. - raise ValueError( - "rdt_produce_weights_batched requires the consumer's issue index `seq`" - ) - gen, is_packer = self._join_share_group(consumer_id, seq) - if not is_packer: - blob = self._await_shared_pack(gen) - else: - try: - blob = self._pack_shared(gen, specs) - except BaseException as e: - self._fail_shared_pack(gen, e) - raise - # A served pull is the third of the four progress signals the stall - # watchdog reads (publish / produce / free / begin_sync): a long sync whose - # consumers pull steadily but slowly must never trip it. - with self._cache_cond: - self._note_progress_locked() - return [blob] - - def _pack_shared(self, gen: _SharedPack, specs: list) -> torch.Tensor: - """Replay the op chains into this generation's slot and publish the blob - to the sharers waiting on it. - - Runs on exactly one call per generation and OUTSIDE ``_cache_cond``: the - pack is GPU work and must not block publishes, frees or other groups. - """ - sliced: list = [] # (byte_off, tensor) - pack_cur = 0 - for name, chain in specs: - t = self._cache[name] - for op, args, kw in chain: - if op not in ALLOWED_OPS: - raise ValueError(f"{name!r}: disallowed op {op!r}") - t = getattr(t, op)(*args, **dict(kw)) - off = (pack_cur + 15) & ~15 - pack_cur = off + t.numel() * t.element_size() - sliced.append((off, t)) - - buffer = self._serve_slot(gen.group, gen.slot, pack_cur) - - # The destination views are a pure function of the packed - # layout, which is byte-identical every sync, so build them once per - # (sharing group, ring slot, layout) and reuse — rebuilding per call cost - # 5.2ms of a 7.5ms 384-spec 235B group. - # - # The key is the LAYOUT, not the spec names: a name can appear in two - # requests with different op chains (one name's copies can split across - # owner-class chunks), and serving the second through the first's views - # would write the wrong bytes with nothing downstream to catch it. - dst_key = ( - gen.group, - gen.slot, - tuple((off, t.dtype, t.shape) for off, t in sliced), - ) - cached = self._pack_dsts.get(dst_key) - if cached is None or cached[0] != buffer.data_ptr(): - dsts = [] - for off, t in sliced: - nb = t.numel() * t.element_size() - dsts.append(buffer[off : off + nb].view(t.dtype).reshape(t.shape)) - self._pack_dsts[dst_key] = (buffer.data_ptr(), dsts) - else: - dsts = cached[1] - torch._foreach_copy_(dsts, [t for _off, t in sliced]) - - blob = buffer[:pack_cur] - with self._cache_cond: - gen.blob = blob - gen.done = True - self._note_progress_locked() - self._cache_cond.notify_all() - return blob - - def shutdown(self) -> None: - with self._cache_cond: - self._cache.clear() - with self._cache_cond: - # Generations hold blob views into the rings, so they go first. - self._gens.clear() - self._sharers.clear() - with self._serve_lock: - self._serve_rings.clear() - # Must go with the rings: these are views INTO them, and the - # data_ptr guard that normally invalidates them cannot tell a freed - # buffer from a new one recycled at the same address. - self._pack_dsts.clear() - - -class ShardedRDTTrainerWeightTransferEngine( - TrainerWeightTransferEngine[ShardedRDTTrainerInitInfo] -): - """Trainer-side engine for the pull-based sharded-RDT backend. - - Lives on every trainer rank. Owns a per-rank `_RDTProducerServer` actor - (the NIXL serve surface). `send_weights` gathers this rank's weights - group-by-group from the `WeightSource`, shares each group into the server - over CUDA IPC, and — on the sender — drives the inference-side handshake so - the workers pull. Non-sender ranks only gather (staying in the collective). - """ - - init_info_cls = ShardedRDTTrainerInitInfo - - def __init__( - self, - *, - client: VLLMWeightSyncClient, - source: WeightSource, - is_sender: bool = True, - init_info: ShardedRDTTrainerInitInfo, - ) -> None: - super().__init__(client=client, source=source, is_sender=is_sender) - self._init_info = init_info - self._server: Any = None # the _RDTProducerServer actor handle - self._server_name: str | None = None - # Group-major metadata / partition, computed at trainer_init. - self._meta: list[ParamMeta] = [] - self._groups: list[list[str]] = [] - # Ownership resolved at trainer_init from the fleet's held names: the - # distinct owner sets, the per-name index into them, this rank's groups - # and its held set. Routing itself is consumer-side; the trainer only - # ships the table. - self._owner_sets: list[list[int]] = [] - self._name_owner_class: list[int] = [] - self._owned_idx: list[int] = [] - self._held_names: set[str] | None = None - # Strong refs to gathered tensors we've shared into the server, keyed by - # group index. CUDA-IPC exports must outlive the importer, so we hold - # them until the server reports the group freed. See send_weights. - self._inflight: dict[int, dict[str, torch.Tensor]] = {} - # [RDT-EXPORT-RING] Reusable packed slots, one per resident group. See - # _pack_group_for_export. Persistent across syncs: freeing and - # reallocating them each sync fragmented the caching allocator badly - # enough to grow reserved memory ~1.5 GB per sync. - self._export_ring: list[torch.Tensor | None] = [] - self._export_ring_args: list[tuple | None] = [] - # Which live group holds which slot, and the slots free to hand out. A - # modular counter is unsafe here: the credit gate bounds the NUMBER of - # unfreed groups, not their order, so it can hand a later group the slot - # a still-live group is being served from. - self._slot_of_group: dict[int, int] = {} - self._free_slots: list[int] = [] - # Set by trainer_init from the server-name all-gather; retained so a - # restarted consumer can be re-initialized without another collective. - self._server_names: list[str] | None = None - - def _rpc(self, method: str, *args: Any) -> Any: - """Call one of the server actor's methods and block for the result. - The single seam through which the engine talks to its server, so tests - can inject a local (non-Ray) fake server.""" - import ray - - return ray.get(getattr(self._server, method).remote(*args)) - - @staticmethod - def _contig_stride(shape) -> tuple: - stride: list[int] = [] - acc = 1 - for s in reversed(list(shape)): - stride.append(acc) - acc *= int(s) - return tuple(reversed(stride)) - - def _pack_group_for_export(self, held: list, slot_idx: int) -> tuple: - """Copy ``held`` into ring slot ``slot_idx``; return ``(storages, views, - refs)`` in the same shape the per-storage path returns, but with ONE - storage. Views are built exactly as ``publish_group`` rebuilds them, so - the two sides cannot disagree about layout. - - Slot safety is not the credit gate alone: that bounds how MANY groups - are unfreed, not the order they free in, and barriers do complete out of - order. The caller takes ``slot_idx`` from a pool keyed by live group - (``_slot_of_group`` / ``_free_slots``); ``_drop_inflight`` returns a slot - only once its group is freed everywhere. - """ - offsets: list[int] = [] - cur = 0 - for _name, t in held: - cur = (cur + 15) & ~15 # 16B keeps every element size we ship aligned - offsets.append(cur) - cur += t.numel() * t.element_size() - need = max(16, (cur + 15) & ~15) - - slot = self._export_ring[slot_idx] - device = held[0][1].device - if slot is None or slot.numel() < need or slot.device != device: - self._export_ring[slot_idx] = self._export_ring_args[slot_idx] = None - slot = torch.empty(need, dtype=torch.uint8, device=device) - self._export_ring[slot_idx] = slot - - ust = slot.untyped_storage() - sid = ust.data_ptr() - reduce_args = self._export_ring_args[slot_idx] - if reduce_args is None: - base = torch.empty(0, dtype=torch.uint8, device=device) - base.set_(ust, 0, (ust.nbytes(),)) - _rebuild, reduce_args = reduce_tensor(base) - self._export_ring_args[slot_idx] = reduce_args - - views: dict[str, tuple] = {} - refs: dict[str, torch.Tensor] = {} - for (name, t), off in zip(held, offsets): - shape, esz = tuple(t.shape), t.element_size() - stride = self._contig_stride(shape) - dst = torch.as_strided(slot.view(t.dtype), shape, stride, off // esz) - dst.copy_(t) - refs[name] = dst - views[name] = ( - sid, - str(t.dtype).split(".")[-1], - list(shape), - list(stride), - off // esz, - ) - # The producer sidecar reads this slot from another process over CUDA - # IPC, while these copies are merely enqueued on this rank's stream; - # nothing orders the two. Without this sync it packs bytes the copy has - # not written yet -- silent corruption, and only for ring-packed tensors. - if slot.is_cuda: - torch.cuda.current_stream(slot.device).synchronize() - return {sid: reduce_args}, views, refs - - def _publish_async(self, group_idx: int, entries): - """Fire publish_group WITHOUT waiting on the RPC (the gather loop - overlaps the publish's server-side rebuild with the next group's - gather) and return a handle that ``_await_publish`` resolves. Ray actor - handle in production; a plain (non-Ray) fake server runs inline.""" - method = self._server.publish_group - remote = getattr(method, "remote", None) - if remote is not None: - return remote(group_idx, entries) - return method(group_idx, entries) - - def _await_publish(self, ref) -> None: - """Resolve one async publish so a server-side rebuild error surfaces at - window depth instead of at end_sync. Publishes carry nothing back — - freed groups flow only through wait_freed/end_sync (one channel; see - publish_group).""" - import ray - - if isinstance(ref, ray.ObjectRef): - ray.get(ref) - - # ---------------- construction ---------------- - - @classmethod - def trainer_init( - cls, - init_info: ShardedRDTTrainerInitInfo, - *, - client: VLLMWeightSyncClient, - source: WeightSource | None = None, - ) -> Self: - if source is None: - raise ValueError( - "Sharded RDT trainer weight transfer requires a WeightSource." - ) - engine = cls( - client=client, - source=source, - is_sender=init_info.is_sender, - init_info=init_info, - ) - - # The VMM-based expandable_segments allocator makes IPC storage opens ~9x - # slower on both export and rebuild, and CUDA-IPC publish is this engine's - # hot path. Frameworks enabling it should disable it around send_weights. - # Only the env var is visible; runtime changes are not introspectable. - import os as _os - - if "expandable_segments:True" in _os.environ.get("PYTORCH_CUDA_ALLOC_CONF", ""): - logger.warning( - "Sharded-RDT trainer: PYTORCH_CUDA_ALLOC_CONF enables " - "expandable_segments; CUDA-IPC weight publishing will be " - "several times slower. Disable expandable segments around " - "weight sync (allocate gather buffers in classic segments)." - ) - - engine._meta = list(source.metadata()) - names = [m.name for m in engine._meta] - engine._groups = layerwise_groups(names) - flat = [n for g in engine._groups for n in g] - if flat != names: - raise ValueError( - "Sharded RDT requires a WeightSource whose metadata order is " - "group-contiguous. Reorder the source so all names sharing a " - "layer index are adjacent." - ) - - world, rank = engine._world_and_rank() - engine._resolve_ownership(world, rank) - engine._spawn_server(sorted(engine._held_names or [])) - - # Every rank's server must exist before the sender's init RPC (the worker - # init calls reserve_serve_buffer back on ALL producer servers). The - # all-gather of server names doubles as that barrier. - server_names = engine._all_gather_server_names(world, rank) - # Retained so a RESTARTED consumer can be re-initialized without another - # all-gather (see get_worker_init_payload). The uuid actor names stay valid - # for the run, because producers never restart. - engine._server_names = server_names - - if engine.is_sender: - worker_init = engine._build_worker_init_info(server_names) - engine.client.init_weight_transfer_engine(asdict(worker_init)) - return engine - - def get_worker_init_payload(self) -> dict: - """The consumer-side init payload, rebuilt on demand. Pure — no collective. - - Reads only retained state, so a restarted inference engine can rejoin at a - sync boundary: the driver can ask any time, including mid-run with every - rank in its own training step. A collective here would deadlock, since the - ranks are not at a matching one. - - Raises: - RuntimeError: called before ``trainer_init`` cached the server names. - - """ - if self._server_names is None: - raise RuntimeError( - "get_worker_init_payload requires trainer_init to have run (the " - "producer server names are gathered there)." - ) - return asdict(self._build_worker_init_info(self._server_names)) - - def _world_and_rank(self) -> tuple[int, int]: - if torch.distributed.is_available() and torch.distributed.is_initialized(): - return torch.distributed.get_world_size(), torch.distributed.get_rank() - return 1, self._init_info.rank - - def _resolve_ownership(self, world: int, rank: int) -> None: - """Resolve which rank holds which name, and this rank's publish plan. - - A source may hold only part of the model — pipeline stages, expert - parallelism, or any mix — so each rank declares its held names and the - fleet all-gathers them. The consumers route per name, so the wire carries - the transposed result: the distinct owner sets, and a per-name index into - them. - - The masks are positional over metadata order, which is why the metadata - digest is checked first: a rank whose names disagree would transpose into - the wrong owners entirely. - """ - assert self.source is not None # guaranteed by trainer_init - num_groups = len(self._groups) - names = [m.name for m in self._meta] - held = self.source.held_names() - if held is None: - held_set = set(names) - else: - held_set = {str(n) for n in held} - unknown = sorted(held_set - set(names)) - if unknown: - raise ValueError( - f"WeightSource.held_names() lists {len(unknown)} name(s) not " - f"in metadata(), e.g. {unknown[:3]}." - ) - if not held_set: - raise ValueError( - "WeightSource.held_names() is empty; a rank with nothing to " - "serve cannot take part in the gather" - ) - - # One collective carries this rank's holdings as a bitmask over metadata - # order, plus the digest of the metadata it indexes into. The digest is - # what makes partial ownership safe: only the sender's metadata reaches - # the consumers, so a rank describing just its own share would silently - # serve the wrong weights. - digest = self._meta_digest() - mask = bytearray((len(names) + 7) // 8) - for i, n in enumerate(names): - if n in held_set: - mask[i >> 3] |= 1 << (i & 7) - per_rank = self._all_gather_owned(world, (digest, bytes(mask))) - mismatched = [r for r, (d, *_rest) in enumerate(per_rank) if d != digest] - if mismatched: - raise ValueError( - f"WeightSource.metadata() disagrees across trainer ranks " - f"(rank {rank} digest {digest}, differing ranks {mismatched[:4]}). " - "Every rank must describe the WHOLE model, even when it holds " - "only some of it." - ) - masks = [m for _d, m in per_rank] - - # Transpose to per-name owners, then dedup into classes. Numbering by - # FIRST APPEARANCE in metadata order keeps it a pure function of - # rank-identical inputs, which is what lets a rejoining consumer rebuild - # the identical table from get_worker_init_payload. - owner_sets: list[list[int]] = [] - class_of_owners: dict[tuple[int, ...], int] = {} - name_owner_class: list[int] = [] - for i, n in enumerate(names): - owners = tuple(r for r, m in enumerate(masks) if m[i >> 3] & (1 << (i & 7))) - if not owners: - raise ValueError( - f"no trainer rank holds {n!r}; every name in metadata() must " - "be held by at least one rank or it can never be served." - ) - ci = class_of_owners.get(owners) - if ci is None: - ci = len(owner_sets) - class_of_owners[owners] = ci - owner_sets.append(list(owners)) - name_owner_class.append(ci) - - self._owner_sets = owner_sets - self._name_owner_class = name_owner_class - # Groups holding anything here: exactly what iteration must cover. - self._owned_idx = [ - gi - for gi in range(num_groups) - if any(n in held_set for n in self._groups[gi]) - ] - self._held_names = held_set - - def _validate_held_yields(self, gi: int, names, tensors) -> None: - """Check ``held_names()`` against what the source actually yields — the - one place both sit side by side. - - Without it, a source that claims a name but yields ``None`` for it dies - by stall watchdog 300s later: consumers route pulls here, the pull - passes the served-names guard, and the cache wait never completes. With - it, that is an immediate error naming the weight. One set lookup per - name per sync.""" - if self._held_names is None: - return - for name, tensor in zip(names, tensors): - if (tensor is None) == (name in self._held_names): - claim = "does not hold" if tensor is not None else "holds" - have = "a real tensor" if tensor is not None else "None" - raise RuntimeError( - f"held_names() disagrees with the yielded tensors: group {gi} " - f"name {name!r} yielded {have} but this rank {claim} it. A " - "WeightSource must yield a real tensor for every held name " - "and None for the rest (see WeightSource.held_names)." - ) - - def _meta_digest(self) -> str: - """Stable digest of this rank's metadata (name order + count).""" - import hashlib - - h = hashlib.sha256() - h.update(f"{len(self._meta)}\n".encode()) - for m in self._meta: - h.update(m.name.encode()) - h.update(b"\n") - return h.hexdigest()[:16] - - def _all_gather_owned(self, world: int, mine: tuple) -> list[tuple]: - """All-gather each rank's (metadata digest, held-name bitmask).""" - if world <= 1 or not ( - torch.distributed.is_available() and torch.distributed.is_initialized() - ): - return [mine] - gathered: list[Any] = [None] * world - torch.distributed.all_gather_object(gathered, mine) - return gathered - - def _all_gather_server_names(self, world: int, rank: int) -> list[str]: - assert self._server_name is not None - if world <= 1 or not ( - torch.distributed.is_available() and torch.distributed.is_initialized() - ): - return [self._server_name] - gathered: list[str | None] = [None] * world - torch.distributed.all_gather_object(gathered, self._server_name) - return [n for n in gathered if n is not None] - - def _spawn_server(self, served_names: list[str]) -> None: - import ray - from ray.util.scheduling_strategies import NodeAffinitySchedulingStrategy - - # The trainer is usually a separate install from the workers, so it gets - # its own check rather than trusting the consumer side's. - check_ray_rdt_version() - - ii = self._init_info - self._server_name = f"vllm_rdt_producer_{uuid.uuid4().hex[:12]}_rk{ii.rank}" - node_id = ray.get_runtime_context().get_node_id() - # Pin the server to this rank's physical GPU: num_gpus=0 so Ray does not - # allocate a second, CUDA_VISIBLE_DEVICES so CUDA IPC to the rank's - # gathered tensors works. max_concurrency > 1 serves pulls while control - # calls are in flight; enable_tensor_transport gives the NIXL serve. - gpu_ids = ray.get_gpu_ids() - # The server is the trainer rank's process twin: forward the env the - # rank runs under (library paths etc.) so it imports torch/vllm the - # same way, then pin it to the rank's physical GPU for CUDA IPC. - import os - - env_vars = { - k: os.environ[k] - for k in ( - "LD_LIBRARY_PATH", - "LD_PRELOAD", - "NCCL_CUMEM_ENABLE", - "VLLM_NCCL_SO_PATH", - "PATH", - ) - if k in os.environ - } - if gpu_ids: - # A num_gpus=0 actor sharing the rank's GPU: Ray would set - # CUDA_VISIBLE_DEVICES="" and hide every GPU, so tell it not to touch - # the var and pin the device ourselves. - env_vars["RAY_EXPERIMENTAL_NOSET_CUDA_VISIBLE_DEVICES"] = "1" - env_vars["CUDA_VISIBLE_DEVICES"] = str(gpu_ids[0]) - runtime_env = {"env_vars": env_vars} if env_vars else {} - server_cls = ray.remote(_RDTProducerServer).options( - name=self._server_name, - namespace=ii.trainer_actor_namespace, - num_cpus=0, - num_gpus=0, - # Thread budget: under EP-local routing every consumer pulls from every - # producer, and with issue-ahead each can park K produce calls in the - # cache-wait — C*K blocked calls. A smaller pool queues the control - # plane behind them and deadlocks: the parked produces wait for a - # publish that can never get a thread (the sync-2 wedge at 235B tp8, - # 16 parked vs 8 threads). Size for the worst case plus control-plane - # slack. - max_concurrency=ii.num_consumers * ii.num_rdt_buffers + 4, - enable_tensor_transport=True, - scheduling_strategy=NodeAffinitySchedulingStrategy( - node_id=node_id, soft=False - ), - runtime_env=runtime_env, - ) - self._server = server_cls.remote( - served_names=served_names, - num_rdt_buffers=ii.num_rdt_buffers, - buffer_presize_gb=ii.buffer_presize_gb, - gather_lookahead=ii.gather_lookahead, - stall_timeout_s=ii.stall_timeout_s, - workers_per_replica=ii.workers_per_replica, - ) - ray.get(self._server.ping.remote()) - # Pre-barrier NIXL warmup: must complete before the server-name - # all-gather so no rank can be inside a collective yet (see warmup_nixl). - ray.get(self._server.warmup_nixl.remote()) - - def _build_worker_init_info(self, server_names: list[str]): - from vllm.distributed.weight_transfer.sharded_rdt_engine import ( - ShardedRDTWeightTransferInitInfo, - ) - - group_lens = [len(g) for g in self._groups] - names = [m.name for m in self._meta] - dtype_names = [str(m.dtype).split(".")[-1] for m in self._meta] - shapes = [list(m.shape) for m in self._meta] - return ShardedRDTWeightTransferInitInfo( - trainer_actor_names=server_names, - trainer_actor_namespace=self._init_info.trainer_actor_namespace, - produce_method_name=PRODUCE_METHOD_NAME, - names=names, - dtype_names=dtype_names, - shapes=shapes, - group_lens=group_lens, - owner_sets=self._owner_sets, - name_owner_class=self._name_owner_class, - num_consumers=self._init_info.num_consumers, - num_rdt_buffers=self._init_info.num_rdt_buffers, - buffer_presize_gb=self._init_info.buffer_presize_gb, - ) - - # ---------------- per-round ---------------- - - def send_weights(self, live_consumer_ids: Collection[int] | None = None) -> None: - """Gather this rank's weights and publish them for the consumers to pull. - - ``live_consumer_ids`` restricts the sync to the consumers still alive; - ``None`` serves the whole provisioned set. The provisioned geometry is - FROZEN for the run — a degraded sync only lowers the live count handed to - ``begin_sync``. Every rank must get the SAME live set, since they share - the gather collectives, so the caller computes it once for all of them. - """ - assert self.source is not None - if live_consumer_ids is None: - live_ids = list(range(self._init_info.num_consumers)) - live_count = len(live_ids) - else: - live_ids = sorted(set(int(c) for c in live_consumer_ids)) - live_count = len(live_ids) - logger.warning( - "[rdt-degraded] serving %d/%d live consumers; every group's " - "free barrier counts to the live total", - live_count, - self._init_info.num_consumers, - ) - self._send_weights_inner(live_count, live_ids) - - def _send_weights_inner(self, live_count: int, live_ids: list[int]) -> None: - if not self.is_sender: - self._run_gather_loop( - update_future=None, live_count=live_count, live_ids=live_ids - ) - return - - self.client.start_weight_update() - - from vllm.distributed.weight_transfer.sharded_rdt_engine import ( - ShardedRDTWeightTransferUpdateInfo, - ) - - empty_update = asdict(ShardedRDTWeightTransferUpdateInfo()) - with ThreadPoolExecutor(max_workers=1) as exe: - # The workers block inside update_weights until they've pulled every - # group, so it runs concurrently with the gather/publish loop. - future = exe.submit(self.client.update_weights, empty_update) - self._run_gather_loop( - update_future=future, live_count=live_count, live_ids=live_ids - ) - future.result() # surface inference-side errors - - self.client.finish_weight_update() - - def _run_gather_loop( - self, update_future, live_count: int, live_ids: list[int] | None = None - ) -> None: - """Gather this rank's weights group-by-group and publish each into the - server over CUDA IPC. A gathered group is published — serveable — - immediately; the loop gates BEFORE the next gather while more than - `gather_lookahead` groups are unfreed (the per-group free barrier: a - credit releases when every live consumer has signaled the group). So - the loop self-paces to the consumers' pull rate with at most - `gather_lookahead + 1` groups resident. Runs on every rank; only the - sender has an `update_future` to fail fast on.""" - assert self.source is not None # guaranteed by trainer_init - # The live IDS ride along with the count: the free barrier needs only - # the count, but the slot-sharing rendezvous needs to know WHICH - # consumers are live. - self._rpc("begin_sync", live_count, live_ids) - # One generator resume per GROUP: `iter_groups` yields (names, tensors) - # per owned group in metadata order. Every owner must reach a group in the - # same order or their shared gather collective mismatches. - groups = self.source.iter_groups() - # Publishes fire without an inline ray.get and are harvested this window - # deep, so the RPC and server-side rebuild overlap the NEXT group's - # gather/export. The window also surfaces a server-side error at depth 2 - # rather than at end_sync. - _PUBLISH_WINDOW = 2 - pending_publish: list = [] - # The memory bound: gate BEFORE gathering while more - # than `bound` groups are unfreed, so at most bound + 1 are resident. The - # count is exact — `_inflight` gains its entry right after each gather and - # only wait_freed/end_sync shrinks it, so a group freed server-side but not - # yet collected still holds its refs and still counts. The publish window - # drains first, so every resident group is pullable while we wait. - # - # 0 is legal and means NO pipelining: the loop waits for group N to be - # freed by every live consumer before gathering N+1, so exactly one group - # is resident and the sync costs the full serialized - # gather + publish + pull + free RTT per group. 1 (the default) overlaps - # N+1's gather with N's pulls. - bound = max(0, self._init_info.gather_lookahead) - - # [RDT-EXPORT-RING] `bound + 1` slots is exactly the residency the credit - # gate enforces. Small storages are packed into a slot and exported once - # per slot instead of once per storage (518 -> 7 cudaIpcGetMemHandle - # calls per rank per sync at 235B, each 0.5-1.5ms). - nslots = bound + 1 - # data_ptr -> (storage weakref, reduce_args), this sync only. `storages` - # is per-group, so a source that reuses a buffer across groups would - # otherwise re-export it every group. The weakref is load-bearing: a - # freed allocation can be reissued at the same address, and serving that - # from a stale handle is silently wrong bytes. - handle_cache: dict[int, tuple] = {} - if len(getattr(self, "_export_ring", [])) != nslots: - self._export_ring = [None] * nslots - self._export_ring_args = [None] * nslots - # Slots are handed out per group and returned by _drop_inflight, so the - # pool starts full every sync. - self._slot_of_group = {} - self._free_slots = list(range(nslots)) - - try: - for gi in self._owned_idx: - group = self._groups[gi] - while len(self._inflight) > bound: - while pending_publish: - self._await_publish(pending_publish.pop(0)) - self._drop_inflight(self._rpc("wait_freed")) - names, tensors = next(groups) - if list(names) != list(group): - raise RuntimeError( - f"WeightSource group yielded {len(names)} names starting " - f"{names[:2]!r} but expected {len(group)} starting " - f"{group[:2]!r}; iteration order must match metadata." - ) - self._validate_held_yields(gi, names, tensors) - # Share each unique STORAGE once and describe every name as an - # as_strided view onto it. ``None`` means a name this rank does not - # hold (a foreign expert); the source keeps it in the list so the - # order check stays rank-uniform, and it is dropped here, before - # the IPC export, matching the sidecar's served_names. - storages: dict[int, tuple] = {} - views: dict[str, tuple] = {} - refs: dict[str, torch.Tensor] = {} - small: list = [] - for name, tensor in zip(names, tensors): - if tensor is None: - continue - tensor = tensor.detach() - if not tensor.is_cuda: - tensor = tensor.cuda() - # Threshold on the STORAGE, not the tensor: a source may hand - # out many small views of one large stacked buffer, and a - # per-tensor test would pack the whole thing, costing +1 - # group of peak memory for a storage already coalesced. - if tensor.untyped_storage().nbytes() <= _EXPORT_RING_MAX_BYTES: - small.append((name, tensor)) - continue - tensor = tensor.contiguous() - refs[name] = tensor # keep the export alive - ust = tensor.untyped_storage() - sid = ust.data_ptr() - if sid not in storages: - cached = handle_cache.get(sid) - if cached is not None and not cached[0].expired(): - reduce_args = cached[1] - else: - base = torch.empty( - 0, dtype=torch.uint8, device=tensor.device - ) - base.set_(ust, 0, (ust.nbytes(),)) - _rebuild, reduce_args = reduce_tensor(base) - handle_cache[sid] = (StorageWeakRef(ust), reduce_args) - storages[sid] = reduce_args - views[name] = ( - sid, - str(tensor.dtype).split(".")[-1], - list(tensor.shape), - list(tensor.stride()), - tensor.storage_offset(), - ) - if small: - # Take a slot no live group holds. The gate above already - # guarantees one exists, so this is a backstop -- but it - # must drain publishes first, like the credit gate. - while not self._free_slots: - while pending_publish: - self._await_publish(pending_publish.pop(0)) - self._drop_inflight(self._rpc("wait_freed")) - slot_idx = self._free_slots.pop(0) - self._slot_of_group[gi] = slot_idx - st, vw, rf = self._pack_group_for_export(small, slot_idx) - storages.update(st) - views.update(vw) - refs.update(rf) - del small - del tensors - # The loop leaves its last `tensor` / `ust` / `base` bound - # through the credit gate and into the next gather, and `base` - # is a whole-storage view -- so one tensor and its entire - # allocation stay alive past `_drop_inflight`, unseen by the - # `_inflight` accounting the residency bound rests on. - # Assignment, not `del`: unbound if no direct-path tensor. - tensor = ust = base = None - if not refs: - # A group with nothing held here cannot occur (every group - # carries replicated names), but publishing an empty group - # would park a credit nobody's pull is waiting on; skip. - continue - # Hold our refs before publishing; drop them only when the - # server reports the group freed (IPC export must outlive import). - self._inflight[gi] = refs - pending_publish.append(self._publish_async(gi, (storages, views))) - while len(pending_publish) >= _PUBLISH_WINDOW: - self._await_publish(pending_publish.pop(0)) - if update_future is not None and update_future.done(): - # update_weights returned/failed early — surface now instead - # of blocking further gathers. - update_future.result() - while pending_publish: - self._await_publish(pending_publish.pop(0)) - freed = self._rpc("end_sync") - self._drop_inflight(freed) - except BaseException as e: - with contextlib.suppress(Exception): - self._rpc("set_gather_error", repr(e)) - self._inflight.clear() - raise - finally: - # Keep the slots, drop the handles: the sidecar's base cache is - # cleared per sync, so a handle must not outlive the sync either. - self._export_ring_args = [None] * len(getattr(self, "_export_ring", [])) - - def _drop_inflight(self, freed_keys: list) -> None: - """Release a freed group's refs and return its export-ring slot.""" - free_slots = getattr(self, "_free_slots", None) - slot_of_group = getattr(self, "_slot_of_group", None) - for k in freed_keys: - gi = int(k) - self._inflight.pop(gi, None) - if slot_of_group is None: - continue - slot = slot_of_group.pop(gi, None) - if slot is not None and free_slots is not None and slot not in free_slots: - free_slots.append(slot) - - # ---------------- misc ---------------- - - def shutdown(self) -> None: - if self._server is None: - return - import ray - - with contextlib.suppress(Exception): - ray.get(self._server.shutdown.remote()) - ray.kill(self._server) - self._server = None - self._inflight.clear() diff --git a/vllm/distributed/weight_transfer/sparse_nccl_engine.py b/vllm/distributed/weight_transfer/sparse_nccl_engine.py deleted file mode 100644 index f9233516a5..0000000000 --- a/vllm/distributed/weight_transfer/sparse_nccl_engine.py +++ /dev/null @@ -1,377 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Sparse NCCL weight transfer engine. - -Sparse patches use checkpoint names, shapes, and flat indices. The model's native -weight loader maps them to rank-local runtime parameters, including TP shards and -packed parameters. -""" - -from collections.abc import Iterable -from concurrent.futures import ThreadPoolExecutor -from contextlib import nullcontext -from dataclasses import asdict, dataclass -from typing import TYPE_CHECKING, ClassVar, Self - -import torch - -if TYPE_CHECKING: - from vllm.config import VllmConfig - from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator - -from vllm.config.weight_transfer import WeightTransferConfig -from vllm.distributed.weight_transfer.base import ( - TrainerInitInfo, - TrainerWeightTransferEngine, - VLLMWeightSyncClient, - WeightSource, - WeightTransferEngine, - WeightTransferUpdateInfo, -) -from vllm.distributed.weight_transfer.nccl_common import ( - NCCLWeightTransferInitInfo, - worker_init_payload, - worker_init_process_group, -) -from vllm.distributed.weight_transfer.nccl_common import ( - trainer_init as open_trainer_endpoint, -) -from vllm.model_executor.model_loader.checkpoint_weight_patch import ( - CheckpointWeightPatch, - load_checkpoint_weight_patches, -) - -__all__ = [ - "SparseWeightPatch", - "SparseNCCLTrainerInitInfo", - "SparseNCCLWeightTransferUpdateInfo", - "SparseNCCLWeightTransferEngine", - "SparseNCCLTrainerWeightTransferEngine", -] - - -@dataclass -class SparseWeightPatch: - """A sparse patch in checkpoint coordinates.""" - - name: str - indices: torch.Tensor - values: torch.Tensor - full_shape: tuple[int, ...] - """Full checkpoint shape.""" - - -@dataclass -class SparseNCCLTrainerInitInfo(TrainerInitInfo): - """Trainer-side init info for the sparse NCCL weight transfer backend. - - Same rendezvous shape as the dense NCCL backend (the sender opens its - endpoint as NCCL rank 0), but with no packed wire params: sparse transfers - are never packed. `backend` is the factory dispatch key.""" - - backend: ClassVar[str] = "sparse_nccl" - - master_address: str - master_port: int - world_size: int - - -@dataclass -class SparseNCCLWeightTransferUpdateInfo(WeightTransferUpdateInfo): - """Update info for the sparse NCCL weight transfer backend.""" - - names: list[str] - dtype_names: list[str] - shapes: list[list[int]] - num_updates_list: list[int] - """Number of sparse entries to receive for each parameter in ``names``.""" - - def __post_init__(self) -> None: - num_params = len(self.names) - if len(self.dtype_names) != num_params: - raise ValueError( - f"`dtype_names` should be of the same size as `names`: " - f"got {len(self.dtype_names)} and {len(self.names)}" - ) - if len(self.shapes) != num_params: - raise ValueError( - f"`shapes` should be of the same size as `names`: " - f"got {len(self.shapes)} and {len(self.names)}" - ) - if len(self.num_updates_list) == 0: - raise ValueError("`num_updates_list` cannot be empty for sparse updates") - if len(self.num_updates_list) != num_params: - raise ValueError( - f"`num_updates_list` should be of the same size as `names`: " - f"got {len(self.num_updates_list)} and {len(self.names)}" - ) - if any(num_updates < 0 for num_updates in self.num_updates_list): - raise ValueError("Sparse `num_updates_list` entries must be non-negative") - - -class SparseNCCLWeightTransferEngine( - WeightTransferEngine[NCCLWeightTransferInitInfo, SparseNCCLWeightTransferUpdateInfo] -): - """Sparse weight transfer engine using NCCL. - - Receives checkpoint-coordinate patches broadcast from the trainer and applies - them through the model's native weight loader. Sparse updates modify initialized - model tensors in place, so the layerwise reload lifecycle is not used. - """ - - init_info_cls = NCCLWeightTransferInitInfo - update_info_cls = SparseNCCLWeightTransferUpdateInfo - supports_draft_weight_update = False - - def __init__( - self, - config: WeightTransferConfig, - vllm_config: "VllmConfig", - device: torch.device, - model: torch.nn.Module, - ) -> None: - super().__init__(config, vllm_config, device, model) - self.model_update_group: PyNcclCommunicator | None = None - - def init_transfer_engine(self, init_info: NCCLWeightTransferInitInfo) -> None: - """Initialize the NCCL process group with the trainer.""" - self.model_update_group = worker_init_process_group( - init_info, self.parallel_config - ) - - def start_weight_update(self) -> None: - """No-op: sparse patches are applied in place, no layerwise reload.""" - pass - - def finish_weight_update(self) -> None: - """No-op: sparse patches are applied in place, no layerwise reload.""" - pass - - def receive_weights(self, update_info: SparseNCCLWeightTransferUpdateInfo) -> None: - """Receive sparse flat-index patches from the trainer and apply them.""" - if self.model_update_group is None: - raise RuntimeError( - "NCCL weight transfer not initialized. " - "Call init_transfer_engine() first." - ) - - # Receive on the communicator's device rather than relying on the - # ambient current device in the RPC thread. - device = self.model_update_group.device - with torch.accelerator.device_index(device.index): - stream = torch.cuda.current_stream(device=device) - patches = [] - for name, dtype_name, shape, num_updates in zip( - update_info.names, - update_info.dtype_names, - update_info.shapes, - update_info.num_updates_list, - strict=True, - ): - dtype = getattr(torch, dtype_name) - indices = torch.empty(num_updates, dtype=torch.int32, device=device) - values = torch.empty(num_updates, dtype=dtype, device=device) - self.model_update_group.broadcast(indices, src=0, stream=stream) - self.model_update_group.broadcast(values, src=0, stream=stream) - patches.append( - CheckpointWeightPatch( - name=name, - shape=tuple(shape), - dtype=dtype, - values=values, - indices=indices, - ) - ) - load_checkpoint_weight_patches(self.model, patches) - - def shutdown(self) -> None: - if self.model_update_group is not None: - self.model_update_group.destroy() - self.model_update_group = None - - -class SparseNCCLTrainerWeightTransferEngine( - TrainerWeightTransferEngine[SparseNCCLTrainerInitInfo] -): - """Trainer-side sparse NCCL weight transfer engine. - - Broadcasts flat-index (indices, values) patches from NCCL rank 0 while the - inference-side `update_weights` runs concurrently on a side thread (the - worker's recvs rendezvous inside the same NCCL broadcasts). `send_weights` - owns a complete one-shot lifecycle. RL infrastructure that owns the generic - client lifecycle can call `send_weight_chunk` between one start and finish. - - Sparse patches differ every round, so they are not a stable `WeightSource`: - the engine takes no `source`, and patches are passed directly to the send - methods. An empty patch list is a no-op. - - Only the designated trainer sender joins the transfer group; other trainer - ranks skip sparse sends. - """ - - init_info_cls = SparseNCCLTrainerInitInfo - - def __init__( - self, - *, - client: VLLMWeightSyncClient, - source: WeightSource | None = None, - is_sender: bool = True, - ) -> None: - # Sparse is a delta backend: it takes per-round patches via - # send_weights, so a `source` would silently never be sent. The - # parameter exists only to match the base/factory signature. - if source is not None: - raise ValueError( - "Sparse NCCL weight transfer takes no WeightSource; pass each " - "round's patches to send_weights(patches) instead." - ) - super().__init__(client=client, source=source, is_sender=is_sender) - self.model_update_group: PyNcclCommunicator | None = None - - @classmethod - def trainer_init( - cls, - init_info: SparseNCCLTrainerInitInfo, - *, - client: VLLMWeightSyncClient, - source: WeightSource | None = None, - ) -> Self: - engine = cls(client=client, source=source, is_sender=init_info.is_sender) - if not engine.is_sender: - return engine - - # Workers sit at rank_offset 1, after the single trainer sender rank 0. - # Sparse transfers are never packed, so the worker keeps the unpacked - # defaults on its init info. - worker_init_info = NCCLWeightTransferInitInfo( - master_address=init_info.master_address, - master_port=init_info.master_port, - rank_offset=1, - world_size=init_info.world_size, - ) - - # The inference workers block inside init_weight_transfer_engine waiting - # for the NCCL rendezvous, so we kick that off on a side thread while we - # open the trainer endpoint (rank 0); both sides must rendezvous together. - with ThreadPoolExecutor(max_workers=1) as exe: - future = exe.submit( - engine.client.init_weight_transfer_engine, - worker_init_payload(worker_init_info), - ) - # Open the trainer endpoint as NCCL rank 0 on the current device - # (the init info satisfies the helper's rendezvous protocol). - engine.model_update_group = open_trainer_endpoint(init_info) - future.result() # surface any inference-side init error - - return engine - - def send_weights(self, patches: Iterable[SparseWeightPatch] | None = None) -> None: - """Broadcast one sparse update through a one-shot lifecycle.""" - patches = self._prepare_patches(patches) - if not patches: - return - - self.client.start_weight_update() - self._broadcast_chunk(patches) - self.client.finish_weight_update() - - def send_weight_chunk( - self, patches: Iterable[SparseWeightPatch] | None = None - ) -> None: - """Broadcast one chunk inside a caller-owned weight update lifecycle.""" - patches = self._prepare_patches(patches) - if not patches: - return - self._broadcast_chunk(patches) - - def _prepare_patches( - self, patches: Iterable[SparseWeightPatch] | None - ) -> list[SparseWeightPatch]: - if not self.is_sender: - return [] - - prepared = list(patches) if patches is not None else [] - if not prepared: - return [] - if self.model_update_group is None: - raise RuntimeError("trainer_init() must be called before sending weights.") - for patch in prepared: - self._validate_patch(patch) - return prepared - - def _broadcast_chunk(self, patches: list[SparseWeightPatch]) -> None: - assert self.model_update_group is not None - device = self.model_update_group.device - device_context = ( - torch.accelerator.device_index(device.index) - if device.type != "cpu" - else nullcontext() - ) - with device_context: - update_info = SparseNCCLWeightTransferUpdateInfo( - names=[patch.name for patch in patches], - dtype_names=[ - str(patch.values.dtype).split(".")[-1] for patch in patches - ], - shapes=[list(patch.full_shape) for patch in patches], - num_updates_list=[patch.indices.numel() for patch in patches], - ) - - # update_weights (workers receive) must run concurrently with the - # trainer-side broadcasts — both rendezvous inside the same NCCL calls. - executor = ThreadPoolExecutor(max_workers=1) - try: - future = executor.submit( - self.client.update_weights, asdict(update_info) - ) - # Surface an RPC that failed before the worker entered NCCL rather - # than broadcasting to a peer that will never receive. - if future.done(): - future.result() - stream = torch.cuda.current_stream() - for patch in patches: - self.model_update_group.broadcast( - patch.indices, src=0, stream=stream - ) - self.model_update_group.broadcast( - patch.values, src=0, stream=stream - ) - future.result() # surface inference-side errors - finally: - # A failed broadcast can leave the RPC blocked in the matching - # receive. Waiting for that thread would hide the original error. - executor.shutdown(wait=False, cancel_futures=True) - self._post_send_sync() - - @staticmethod - def _validate_patch(patch: SparseWeightPatch) -> None: - """Reject a malformed patch before starting the NCCL transfer.""" - if patch.full_shape is None: - raise ValueError(f"Sparse patch requires full_shape: {patch.name}") - if patch.indices.dtype != torch.int32: - raise ValueError( - f"Sparse weight updates require int32 indices: {patch.name}" - ) - if patch.indices.ndim != 1 or patch.values.ndim != 1: - raise ValueError( - f"Sparse weight patches must be 1D flattened updates: {patch.name}" - ) - if patch.indices.numel() != patch.values.numel(): - raise ValueError( - f"`indices` and `values` must have matching lengths for {patch.name}" - ) - - def _post_send_sync(self) -> None: - """Wait for the broadcasts to land before returning, so a caller may - rebuild or free the patch tensors as soon as a send method returns rather - than relying on same-stream ordering. See - `NCCLTrainerWeightTransferEngine._post_send_sync` for why there is no - cross-rank barrier.""" - if torch.cuda.is_available(): - torch.cuda.current_stream().synchronize() - - def shutdown(self) -> None: - if self.model_update_group is not None: - self.model_update_group.destroy() - self.model_update_group = None diff --git a/vllm/envs.py b/vllm/envs.py index e7b0a74319..0f43b25a53 100755 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -2178,10 +2178,6 @@ environment_variables: dict[str, Callable[[], Any]] = { "VLLM_XPU_INC_WNA16_BACKEND": env_with_choices( "VLLM_XPU_INC_WNA16_BACKEND", "auto", ["auto", "ark", "w4a16", "w4a8"] ), - # Enable simple KV offload. - "VLLM_USE_SIMPLE_KV_OFFLOAD": lambda: bool( - int(os.getenv("VLLM_USE_SIMPLE_KV_OFFLOAD", "0")) - ), # Direct DCP ops default on when applicable; set to 1 to enforce or 0 to disable. "VLLM_USE_DIRECT_DCP_A2A": lambda: maybe_convert_bool( os.getenv("VLLM_USE_DIRECT_DCP_A2A") diff --git a/vllm/lora/ops/torch_ops/__init__.py b/vllm/lora/ops/torch_ops/__init__.py deleted file mode 100644 index 89865af4e9..0000000000 --- a/vllm/lora/ops/torch_ops/__init__.py +++ /dev/null @@ -1,20 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from vllm.lora.ops.torch_ops.lora_ops import ( - bgmv_expand, # noqa: F401 - bgmv_expand_slice, - bgmv_shrink, - sgmv_expand, - sgmv_expand_slice, - sgmv_shrink, -) - -__all__ = [ - "bgmv_expand", - "bgmv_expand_slice", - "bgmv_shrink", - "sgmv_expand", - "sgmv_expand_slice", - "sgmv_shrink", -] diff --git a/vllm/lora/ops/torch_ops/lora_ops.py b/vllm/lora/ops/torch_ops/lora_ops.py deleted file mode 100644 index 4fc6248d54..0000000000 --- a/vllm/lora/ops/torch_ops/lora_ops.py +++ /dev/null @@ -1,128 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -import torch - - -def sgmv_expand( - inputs: torch.Tensor, - lora_b_weights: torch.Tensor, - output_tensor: torch.Tensor, - b_seq_start_loc: torch.Tensor, - seq_len_tensor: torch.Tensor, - lora_indices_tensor: torch.Tensor, - batches: int, - max_seq_length: int, - token_nums: int, - add_inputs: bool = False, -): - exploded_indices = torch.repeat_interleave(lora_indices_tensor, seq_len_tensor) - - bgmv_expand(inputs, lora_b_weights, output_tensor, exploded_indices, add_inputs) - - -def bgmv_expand( - inputs: torch.Tensor, - lora_b_weights: torch.Tensor, - output_tensor: torch.Tensor, - lora_indices_tensor: torch.Tensor, - add_inputs: bool = True, -): - selected_loras = lora_b_weights[lora_indices_tensor].to(dtype=output_tensor.dtype) - if len(selected_loras.shape) == 4: - selected_loras = selected_loras.squeeze(dim=1) - inputs = inputs.to(dtype=output_tensor.dtype) - outputs = torch.einsum("bi, boi -> bo", inputs, selected_loras) - - limit = output_tensor.shape[0] - if outputs.shape[0] == 1 and output_tensor.shape[0] != 1: - limit = 1 - - # LoRA adapter and model may add different amounts of padding to output - common_len = min(outputs.shape[1], output_tensor.shape[1]) - - if add_inputs: - output_tensor[:, :common_len] += outputs[:limit, :common_len] - else: - output_tensor[:, :common_len] = outputs[:limit, :common_len] - - -def sgmv_shrink( - inputs: torch.Tensor, - lora_a_weights: torch.Tensor, - output_tensor: torch.Tensor, - b_seq_start_loc: torch.Tensor, - seq_len_tensor: torch.Tensor, - lora_indices_tensor: torch.Tensor, - batches: int, - max_seq_length: int, - token_nums: int, - scaling: float, -): - exploded_indices = torch.repeat_interleave(lora_indices_tensor, seq_len_tensor) - - bgmv_shrink(inputs, lora_a_weights, output_tensor, exploded_indices, scaling) - - -def bgmv_shrink( - inputs: torch.Tensor, - lora_b_weights: torch.Tensor, - output_tensor: torch.Tensor, - lora_indices_tensor: torch.Tensor, - scaling: float = 1.0, -): - selected_loras = lora_b_weights[lora_indices_tensor].to(dtype=output_tensor.dtype) - if len(selected_loras.shape) == 4: - selected_loras = selected_loras.squeeze(dim=1) - inputs = inputs.to(dtype=output_tensor.dtype) - outputs = torch.einsum("bi, boi -> bo", inputs, selected_loras) - - output_tensor[:, : outputs.shape[1]] = scaling * outputs[:] - - -def sgmv_expand_slice( - inputs: torch.Tensor, - lora_b_weights: torch.Tensor, - output_tensor: torch.Tensor, - b_seq_start_loc: torch.Tensor, - seq_len_tensor: torch.Tensor, - lora_indices_tensor: torch.Tensor, - batches: int, - max_seq_length: int, - token_nums: int, - slice_offset: int, - slice_size: int, - add_inputs: bool = False, -): - exploded_indices = torch.repeat_interleave(lora_indices_tensor, seq_len_tensor) - - bgmv_expand_slice( - inputs, - lora_b_weights, - output_tensor, - exploded_indices, - slice_offset, - slice_size, - add_inputs, - ) - - -def bgmv_expand_slice( - inputs: torch.Tensor, - lora_b_weights: torch.Tensor, - output_tensor: torch.Tensor, - lora_indices_tensor: torch.Tensor, - slice_offset: int, - slice_size: int, - add_inputs: bool = True, -): - selected_loras = lora_b_weights[lora_indices_tensor].to(dtype=output_tensor.dtype) - inputs = inputs.to(dtype=output_tensor.dtype) - if len(selected_loras.shape) == 4: - selected_loras = selected_loras.squeeze(dim=1) - outputs = torch.einsum("bi, boi -> bo", inputs, selected_loras) - - if add_inputs: - output_tensor[:, slice_offset : slice_offset + slice_size] += outputs[:] - else: - output_tensor[:, slice_offset : slice_offset + slice_size] = outputs[:] diff --git a/vllm/lora/ops/triton_ops/__init__.py b/vllm/lora/ops/triton_ops/__init__.py index 687170b305..31047e95e7 100644 --- a/vllm/lora/ops/triton_ops/__init__.py +++ b/vllm/lora/ops/triton_ops/__init__.py @@ -2,32 +2,14 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from vllm.lora.ops.triton_ops.fused_moe_lora_fp8_op import ( - fused_moe_lora_expand_fp8, - fused_moe_lora_fp8, - fused_moe_lora_shrink_fp8, -) -from vllm.lora.ops.triton_ops.fused_moe_lora_op import ( - fused_moe_lora, - fused_moe_lora_expand, - fused_moe_lora_shrink, -) -from vllm.lora.ops.triton_ops.lora_expand_fp8_op import lora_expand_fp8 +from vllm.lora.ops.triton_ops.fused_moe_lora_op import fused_moe_lora from vllm.lora.ops.triton_ops.lora_expand_op import lora_expand from vllm.lora.ops.triton_ops.lora_kernel_metadata import LoRAKernelMeta -from vllm.lora.ops.triton_ops.lora_shrink_fp8_op import lora_shrink_fp8 from vllm.lora.ops.triton_ops.lora_shrink_op import lora_shrink __all__ = [ - "lora_expand", - "lora_expand_fp8", - "lora_shrink", - "lora_shrink_fp8", - "LoRAKernelMeta", "fused_moe_lora", - "fused_moe_lora_shrink", - "fused_moe_lora_expand", - "fused_moe_lora_fp8", - "fused_moe_lora_shrink_fp8", - "fused_moe_lora_expand_fp8", + "lora_expand", + "lora_shrink", + "LoRAKernelMeta", ] diff --git a/vllm/lora/ops/triton_ops/fused_moe_lora_fp8_op.py b/vllm/lora/ops/triton_ops/fused_moe_lora_fp8_op.py deleted file mode 100644 index 780d4a0d70..0000000000 --- a/vllm/lora/ops/triton_ops/fused_moe_lora_fp8_op.py +++ /dev/null @@ -1,895 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - - -from typing import List # noqa: UP035 - -import torch - -from vllm.distributed import ( - tensor_model_parallel_all_gather, - tensor_model_parallel_all_reduce, -) -from vllm.lora.ops.triton_ops.utils import supports_pdl -from vllm.triton_utils import tl, triton -from vllm.utils.torch_utils import direct_register_custom_op - - -@triton.jit -def _get_lora_id( - lora_ids, - token_lora_mapping_ptr, - lora_idx, - pid_m, - top_k_num, - naive_block_assignment: tl.constexpr, -): - """Returns lora_id.""" - if naive_block_assignment: - token_idx = pid_m // top_k_num - return tl.load(token_lora_mapping_ptr + token_idx) - else: - return tl.load(lora_ids + lora_idx) - - -@triton.jit -def _get_expert_id( - expert_ids_ptr, - lora_id, - pid_m, - stride_el, - max_loras, - naive_block_assignment: tl.constexpr, -): - """Returns expert_id.""" - if naive_block_assignment: - return tl.load(expert_ids_ptr + pid_m) - else: - ind = lora_id * stride_el + pid_m - return tl.load(expert_ids_ptr + ind, ind < max_loras * stride_el, -1) - - -@triton.jit -def _get_token_offs( - sorted_token_ids_ptr, - lora_id, - pid_m, - offs, - stride_tl, - max_loras, - num_valid_tokens, - naive_block_assignment: tl.constexpr, - BLOCK_SIZE_M: tl.constexpr, -): - """Returns token offsets.""" - if naive_block_assignment: - return tl.where(offs == 0, pid_m, num_valid_tokens) - else: - offs_token_id = pid_m * BLOCK_SIZE_M + offs - token_ind = stride_tl * lora_id + offs_token_id - return tl.load( - sorted_token_ids_ptr + token_ind, token_ind < max_loras * stride_tl, 0 - ) - - -_LORA_PTR_DICT: dict[tuple[int, ...], torch.tensor] = {} - - -def _get_ptr(lora_weights: list[torch.Tensor], device: torch.device): - """`_LORA_PTR_DICT` collects the required information during `profile_run`, - After this, it remains constant and subsequent usage is through LUT. - Refer to: - https://github.com/triton-lang/triton/blob/release/3.1.x/python/tutorials/08-grouped-gemm.py - """ - key = tuple(lora_weight.data_ptr() for lora_weight in lora_weights) - - if (ptr_tensor := _LORA_PTR_DICT.get(key)) is not None: - return ptr_tensor - - tensor_ptrs = [] - for lora_weight in lora_weights: - tensor_ptrs.append(lora_weight.data_ptr()) - ptr_tensor = torch.tensor(tensor_ptrs, device=device, dtype=torch.uint64) - - _LORA_PTR_DICT[key] = ptr_tensor - return _LORA_PTR_DICT.get(key) - - -def _adjust_kernel_inputs( - num_active_loras: int, - sorted_token_ids: torch.Tensor | None, - expert_ids: torch.Tensor, -): - """Helper function to adjust kernel inputs when sorted_token_ids is None.""" - if sorted_token_ids is None: - stride_tl = 0 - stride_el = 0 - grid_lora_dim = 1 - else: - stride_tl = sorted_token_ids.stride(0) - stride_el = expert_ids.stride(0) - grid_lora_dim = num_active_loras - return grid_lora_dim, stride_tl, stride_el - - -@triton.jit( - do_not_specialize=[ - "num_valid_tokens", - "EM", - "stride_tl", - "stride_el", - "slice_a_size", - "slice_c_size", - ] -) -def _fused_moe_lora_kernel_fp8( - a_ptr, - b_ptr, - c_ptr, - a_scale_ptr, - b_scale_ptr, - topk_weights_ptr, - sorted_token_ids_ptr, - expert_ids_ptr, - num_tokens_post_padded_ptr, - token_lora_mapping_ptr, - # Matrix dimensions - N, - K, - EM, - num_valid_tokens, - num_experts, - top_k_num, - lora_ids, - adapter_enabled, - max_loras, # <<< PR2: rename, used for masks when grid axis-2 != max_loras - # The stride variables represent how much to increase the ptr by when - # moving by 1 element in a particular dimension. E.g. `stride_am` is - # how much to increase `a_ptr` by to get the element one row down - # (A has M rows). - stride_am, - stride_ak, - stride_bl, - stride_be, - stride_bk, - stride_bn, - stride_cm, - stride_cn, - stride_tl, - stride_el, - stride_asm, - stride_ask, - stride_bsl, - stride_bse, - stride_bsk, - stride_bsn, - # block size for block-wise quantization - group_n: tl.constexpr, - group_k: tl.constexpr, - slice_a_size, - slice_c_size, - # Meta-parameters - num_slice_a: tl.constexpr, - num_slice_c: tl.constexpr, - # top_k_num or 1 depending on input token - # is expanded by top_k or not - token_mapping_factor: tl.constexpr, - # whether use naive block assignment - naive_block_assignment: tl.constexpr, - MUL_ROUTED_WEIGHT: tl.constexpr, - ADD_INPUTS: tl.constexpr, - USE_B_L2_CACHE: tl.constexpr, # new, enable .ca load for B - BLOCK_SIZE_M: tl.constexpr, - BLOCK_SIZE_N: tl.constexpr, - BLOCK_SIZE_K: tl.constexpr, - GROUP_SIZE_M: tl.constexpr, - SPLIT_K: tl.constexpr, - USE_GDC: tl.constexpr, - launch_pdl: tl.constexpr, - IS_PRIMARY: tl.constexpr, - use_fp8_w8a8: tl.constexpr, - use_int8_w8a8: tl.constexpr, - use_int8_w8a16: tl.constexpr, - per_channel_quant: tl.constexpr, -): - pid = tl.program_id(axis=0) - slice_id = tl.program_id(axis=1) - grid_k = tl.cdiv(K, BLOCK_SIZE_K * SPLIT_K) - - # calculate pid_m,pid_n - lora_idx = tl.program_id(axis=2) - pid_sk = pid % SPLIT_K - pid_m_n = pid // SPLIT_K - num_pid_m = tl.cdiv(EM, BLOCK_SIZE_M) - num_pid_n = tl.cdiv(N, BLOCK_SIZE_N) - - num_pid_in_group = GROUP_SIZE_M * num_pid_n - group_id = pid_m_n // num_pid_in_group - first_pid_m = group_id * GROUP_SIZE_M - group_size_m = min(num_pid_m - first_pid_m, GROUP_SIZE_M) - pid_m = first_pid_m + ((pid_m_n % num_pid_in_group) % group_size_m) - pid_n = (pid_m_n % num_pid_in_group) // group_size_m - - offs = tl.arange(0, BLOCK_SIZE_M).to(tl.int64) - - # Get lora_id - lora_id = _get_lora_id( - lora_ids, - token_lora_mapping_ptr, - lora_idx, - pid_m, - top_k_num, - naive_block_assignment, - ) - if lora_id == -1: - return - moe_enabled = tl.load(adapter_enabled + lora_id) - if moe_enabled == 0: - return - if lora_id >= max_loras: - return - - # Non-naive only: check num_tokens_post_padded - if not naive_block_assignment: - num_tokens_post_padded = tl.load(num_tokens_post_padded_ptr + lora_id) - if pid_m * BLOCK_SIZE_M >= num_tokens_post_padded: - return - - # Get expert_id - expert_id = _get_expert_id( - expert_ids_ptr, - lora_id, - pid_m, - stride_el, - max_loras, - naive_block_assignment, - ) - if expert_id == -1: - return - - # Get token offsets - offs_token = _get_token_offs( - sorted_token_ids_ptr, - lora_id, - pid_m, - offs, - stride_tl, - max_loras, - num_valid_tokens, - naive_block_assignment, - BLOCK_SIZE_M, - ) - # get a_ptr,b_ptr,c_ptr - cur_a_ptr = a_ptr + (slice_id % num_slice_a) * slice_a_size - cur_b_ptr = tl.load(b_ptr + slice_id).to(tl.pointer_type(c_ptr.dtype.element_ty)) - cur_c_ptr = c_ptr + (slice_id % num_slice_c) * slice_c_size - - # remove modulo wrap-around - offs_bn = pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N).to(tl.int32) - offs_k = pid_sk * BLOCK_SIZE_K + tl.arange(0, BLOCK_SIZE_K) - token_mask = offs_token < num_valid_tokens - - # get a_ptrs,b_ptrs - a_ptrs = cur_a_ptr + ( - offs_token[:, None] // token_mapping_factor * stride_am - + offs_k[None, :] * stride_ak - ) - - b_ptrs = ( - cur_b_ptr - + lora_id * stride_bl - + expert_id * stride_be - + offs_k[:, None] * stride_bk - + offs_bn[None, :] * stride_bn - ) - - if USE_GDC and IS_PRIMARY: - # GDC launch dependents hints the runtime system to launch dependent kernels. - tl.extra.cuda.gdc_launch_dependents() - - # accumulator - accumulator = tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_N), dtype=tl.float32) - - if USE_GDC and not IS_PRIMARY: - tl.extra.cuda.gdc_wait() - - for k in range(0, grid_k): - k_remaining = K - k * (BLOCK_SIZE_K * SPLIT_K) - # GDC wait waits for ALL programs in the prior kernel to complete - # before continuing. - # pre-fetch lora weight - # add (offs_bn < N) mask; optional .ca for B - b_mask = (offs_k[:, None] < k_remaining) & (offs_bn[None, :] < N) - if USE_B_L2_CACHE: - b = tl.load(b_ptrs, mask=b_mask, other=0.0, cache_modifier=".ca") - else: - b = tl.load(b_ptrs, mask=b_mask, other=0.0) - - if USE_GDC and not IS_PRIMARY: - tl.extra.cuda.gdc_wait() - a = tl.load( - a_ptrs, - mask=token_mask[:, None] & (offs_k[None, :] < k_remaining), - other=0.0, - ) - accumulator += tl.dot(a, b) - # Advance the ptrs to the next K block. - a_ptrs += BLOCK_SIZE_K * SPLIT_K * stride_ak - b_ptrs += BLOCK_SIZE_K * SPLIT_K * stride_bk - - if MUL_ROUTED_WEIGHT: - moe_weight = tl.load(topk_weights_ptr + offs_token, mask=token_mask, other=0.0) - accumulator = accumulator * moe_weight[:, None] - accumulator = accumulator.to(c_ptr.dtype.element_ty) - # Write back the block of the output - offs_cn = pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N) - c_ptrs = cur_c_ptr + stride_cm * offs_token[:, None] + stride_cn * offs_cn[None, :] - c_mask = token_mask[:, None] & (offs_cn[None, :] < N) - - if SPLIT_K == 1: - if ADD_INPUTS: - prev = tl.load(c_ptrs, mask=c_mask, other=0.0) - tl.store(c_ptrs, prev + accumulator, mask=c_mask) - else: - tl.store(c_ptrs, accumulator, mask=c_mask) - else: - tl.atomic_add(c_ptrs, accumulator, mask=c_mask, sem="relaxed") - - -@torch.inference_mode() -def _fused_moe_lora_shrink_fp8( - a_intermediate_cache1: torch.Tensor, - # (num_slices, num_tokens, top_k_num, max_lora_rank) - qcurr_hidden_states: torch.Tensor, # (num_tokens, K,) - lora_a_stacked: list[ - torch.Tensor - ], # [(max_loras, num_experts, max_lora_rank, K,),...] - topk_weights: torch.Tensor, # (num_tokens, top_k_num) - sorted_token_ids: torch.Tensor | None, # (max_loras, _) - expert_ids: torch.Tensor, # (max_loras, _ ,) or (num_tokens * top_k,) - num_tokens_post_padded: torch.Tensor | None, # (max_loras, ) - token_lora_mapping: torch.Tensor, - top_k_num: int, - lora_ids: torch.Tensor, - adapter_enabled: torch.Tensor, - ## adding for kernel - device: torch.device, - N: int, - M: int, - EM: int, - K: int, - num_tokens: int, - num_experts: int, - num_slices: int, - block_size_m: int, - block_size_n: int, - block_size_k: int, - group_size_m: int, - num_warps: int, - num_stages: int, - split_k: int, - num_active_loras: int, - lora_a_scale_stacked: list[torch.Tensor], - mul_routed_weight: bool = False, - use_gdc: bool = False, - act_scale: torch.Tensor | None = None, - use_fp8_w8a8: bool = False, - use_int8_w8a8: bool = False, - use_int8_w8a16: bool = False, - per_channel_quant: bool = False, - block_shape: List[int] | None = None, # noqa: UP006, UP007 -) -> None: - if use_fp8_w8a8 or use_int8_w8a8: - assert lora_a_scale_stacked is not None, ( - "lora_a_scale_stacked must be provided for w8a8 quantization" - ) - assert block_shape is None or triton.cdiv( - lora_a_stacked[0].size(-2), block_shape[0] - ) == lora_a_scale_stacked[0].size(-2), ( - "Incompatible block shape for lora_a_scale_stacked.size(-2) " - ) - assert block_shape is None or triton.cdiv( - lora_a_stacked[0].size(-1), block_shape[1] - ) == lora_a_scale_stacked[0].size(-1), ( - "Incompatible block shape for lora_a_scale_stacked.size(-1) " - ) - elif use_int8_w8a16: - assert lora_a_scale_stacked is not None, ( - "lora_a_scale_stacked must be provided for w8a16 quantization" - ) - assert block_shape is None or block_shape[0] == 0, ( - "Block shape for activation must be 0 for w8a16" - ) - else: - assert act_scale is None - assert lora_a_scale_stacked is None - - if block_shape is not None: - block_size_k = min(block_size_k, min(block_shape[0], block_shape[1])) - - if lora_a_scale_stacked is not None: - b_scale_ptr = _get_ptr(lora_a_scale_stacked, device) - w1_lora_a_scale_stacked = lora_a_scale_stacked[0] - - w1_lora_a_stacked = lora_a_stacked[0] - shrink_config = { - "BLOCK_SIZE_M": block_size_m, - "BLOCK_SIZE_N": block_size_n, - "BLOCK_SIZE_K": block_size_k, - "GROUP_SIZE_M": group_size_m, - "num_warps": num_warps, - "num_stages": num_stages, - "SPLIT_K": split_k, - "USE_GDC": use_gdc, - "launch_pdl": use_gdc, # triton kernel metadata - } - - b_ptr = _get_ptr(lora_a_stacked, device) - - grid_lora_dim, stride_tl, stride_el = _adjust_kernel_inputs( - num_active_loras, sorted_token_ids, expert_ids - ) - - grid = lambda META: ( - split_k - * triton.cdiv(EM, META["BLOCK_SIZE_M"]) - * triton.cdiv(N, META["BLOCK_SIZE_N"]), - len(lora_a_stacked), - grid_lora_dim, - ) - _fused_moe_lora_kernel_fp8[grid]( - qcurr_hidden_states, - b_ptr, - a_intermediate_cache1, - act_scale, - b_scale_ptr if lora_a_scale_stacked is not None else None, - topk_weights, - sorted_token_ids, - expert_ids, - num_tokens_post_padded, - token_lora_mapping, - N, - K, - EM, - num_tokens, - num_experts, - top_k_num, - lora_ids, - adapter_enabled, - lora_a_stacked[0].shape[0], - qcurr_hidden_states.stride(0), - qcurr_hidden_states.stride(1), - w1_lora_a_stacked.stride(0), - w1_lora_a_stacked.stride(1), - w1_lora_a_stacked.stride(3), - w1_lora_a_stacked.stride(2), - a_intermediate_cache1.stride(2), - a_intermediate_cache1.stride(3), - stride_tl, - stride_el, - act_scale.stride(0) if act_scale is not None and act_scale.ndim == 2 else 0, - act_scale.stride(1) if act_scale is not None and act_scale.ndim == 2 else 0, - w1_lora_a_scale_stacked.stride(0) - if lora_a_scale_stacked is not None and w1_lora_a_scale_stacked.ndim >= 2 - else 0, - w1_lora_a_scale_stacked.stride(1) - if lora_a_scale_stacked is not None and w1_lora_a_scale_stacked.ndim >= 2 - else 0, - w1_lora_a_scale_stacked.stride(3) - if lora_a_scale_stacked is not None and w1_lora_a_scale_stacked.ndim == 4 - else 0, - w1_lora_a_scale_stacked.stride(2) - if lora_a_scale_stacked is not None and w1_lora_a_scale_stacked.ndim == 4 - else 0, - 0 if block_shape is None else block_shape[0], - 0 if block_shape is None else block_shape[1], - slice_a_size=qcurr_hidden_states.numel(), - slice_c_size=a_intermediate_cache1.numel() // num_slices, - num_slice_a=1, - num_slice_c=num_slices, - token_mapping_factor=1 if mul_routed_weight else top_k_num, - naive_block_assignment=sorted_token_ids is None, - MUL_ROUTED_WEIGHT=False, - ADD_INPUTS=False, - USE_B_L2_CACHE=True, # new - IS_PRIMARY=True, - use_fp8_w8a8=use_fp8_w8a8, - use_int8_w8a8=use_int8_w8a8, - use_int8_w8a16=use_int8_w8a16, - per_channel_quant=per_channel_quant, - **shrink_config, - ) - - -@torch.inference_mode() -def _fused_moe_lora_expand_fp8( - output: torch.Tensor, # (num_tokens, top_k_num, N*len(lora_a_stacked),) - a_intermediate_cache1: torch.Tensor, # (num_slices, M, top_k_num, max_lora_rank) - lora_b_stacked: list[ - torch.Tensor - ], # [(max_loras, num_experts, max_lora_rank, K,),...] - topk_weights: torch.Tensor, # (num_tokens, top_k_num) - sorted_token_ids: torch.Tensor | None, # (max_loras, _) - expert_ids: torch.Tensor, # (max_loras, _ ,) or (num_tokens * top_k,) - num_tokens_post_padded: torch.Tensor | None, # (max_loras, ) - token_lora_mapping: torch.Tensor, - top_k_num: int, - lora_ids: torch.Tensor, - adapter_enabled: torch.Tensor, - ## adding for kernel - device: torch.device, - N: int, - M: int, - EM: int, - K: int, - num_tokens: int, - num_experts: int, - num_slices: int, - max_lora_rank: int, - w1_output_dim_size: int, - block_size_m: int, - block_size_n: int, - block_size_k: int, - group_size_m: int, - num_warps: int, - num_stages: int, - split_k: int, - num_active_loras: int, - lora_b_scale_stacked: list[torch.Tensor], - mul_routed_weight: bool = False, - offset: int = 0, - use_gdc: bool = False, - act_scale: torch.Tensor | None = None, - use_fp8_w8a8: bool = False, - use_int8_w8a8: bool = False, - use_int8_w8a16: bool = False, - per_channel_quant: bool = False, - block_shape: List[int] | None = None, # noqa: UP006, UP007 -) -> None: - if use_fp8_w8a8 or use_int8_w8a8: - assert lora_b_scale_stacked is not None, ( - "lora_b_scale_stacked must be provided for w8a8 quantization" - ) - assert block_shape is None or triton.cdiv( - lora_b_stacked[0].size(-2), block_shape[0] - ) == lora_b_scale_stacked[0].size(-2), ( - "Incompatible block shape for lora_b_scale_stacked.size(-2) " - ) - assert block_shape is None or triton.cdiv( - lora_b_stacked[0].size(-1), block_shape[1] - ) == lora_b_scale_stacked[0].size(-1), ( - "Incompatible block shape for lora_b_scale_stacked.size(-1) " - ) - elif use_int8_w8a16: - assert lora_b_scale_stacked is not None, ( - "lora_b_scale_stacked must be provided for w8a16 quantization" - ) - assert block_shape is None or block_shape[0] == 0, ( - "Block shape for activation must be 0 for w8a16" - ) - else: - assert act_scale is None - assert lora_b_scale_stacked is None - - if lora_b_scale_stacked is not None: - b_scale_ptr = _get_ptr(lora_b_scale_stacked, device) - w1_lora_b_scale_stacked = lora_b_scale_stacked[0] - - if block_shape is not None: - block_size_k = min(block_size_k, min(block_shape[0], block_shape[1])) - - b_ptr = _get_ptr(lora_b_stacked, device) - K = max_lora_rank - N = w1_output_dim_size - - w1_lora_b_stacked = lora_b_stacked[0] - - a_intermediate_cache1 = a_intermediate_cache1.view( - -1, a_intermediate_cache1.shape[3] - ) - - expand_config = { - "BLOCK_SIZE_M": block_size_m, - "BLOCK_SIZE_N": block_size_n, - "BLOCK_SIZE_K": block_size_k, - "GROUP_SIZE_M": group_size_m, - "num_warps": num_warps, - "num_stages": num_stages, - "SPLIT_K": 1, # Set split_k = 1 for expand calls - "USE_GDC": use_gdc, - "launch_pdl": use_gdc, # triton kernel metadata - } - - grid_lora_dim, stride_tl, stride_el = _adjust_kernel_inputs( - num_active_loras, sorted_token_ids, expert_ids - ) - - grid = lambda META: ( - triton.cdiv(EM, META["BLOCK_SIZE_M"]) * triton.cdiv(N, META["BLOCK_SIZE_N"]), - len(lora_b_stacked), - grid_lora_dim, - ) - - # Fast path: directly accumulate into the corresponding slice interval of output. - out_view = output[:, :, offset : offset + num_slices * N] - slice_c_size = N * out_view.stride(2) - - _fused_moe_lora_kernel_fp8[grid]( - a_intermediate_cache1, - b_ptr, - out_view, - act_scale, - b_scale_ptr if lora_b_scale_stacked is not None else None, - topk_weights, - sorted_token_ids, - expert_ids, - num_tokens_post_padded, - token_lora_mapping, - N, - K, - EM, - num_tokens, - num_experts, - top_k_num, - lora_ids, - adapter_enabled, - lora_b_stacked[0].shape[0], - a_intermediate_cache1.stride(0), - a_intermediate_cache1.stride(1), - w1_lora_b_stacked.stride(0), - w1_lora_b_stacked.stride(1), - w1_lora_b_stacked.stride(3), - w1_lora_b_stacked.stride(2), - out_view.stride(1), - out_view.stride(2), - stride_tl, - stride_el, - act_scale.stride(0) if act_scale is not None and act_scale.ndim == 2 else 0, - act_scale.stride(1) if act_scale is not None and act_scale.ndim == 2 else 0, - w1_lora_b_scale_stacked.stride(0) - if lora_b_scale_stacked is not None and w1_lora_b_scale_stacked.ndim >= 2 - else 0, - w1_lora_b_scale_stacked.stride(1) - if lora_b_scale_stacked is not None and w1_lora_b_scale_stacked.ndim >= 2 - else 0, - w1_lora_b_scale_stacked.stride(3) - if lora_b_scale_stacked is not None and w1_lora_b_scale_stacked.ndim == 4 - else 0, - w1_lora_b_scale_stacked.stride(2) - if lora_b_scale_stacked is not None and w1_lora_b_scale_stacked.ndim == 4 - else 0, - 0 if block_shape is None else block_shape[0], - 0 if block_shape is None else block_shape[1], - slice_a_size=a_intermediate_cache1.numel() // num_slices, - slice_c_size=slice_c_size, - num_slice_a=num_slices, - num_slice_c=num_slices, - token_mapping_factor=1, - naive_block_assignment=sorted_token_ids is None, - MUL_ROUTED_WEIGHT=mul_routed_weight, - ADD_INPUTS=True, - USE_B_L2_CACHE=True, # new - IS_PRIMARY=False, - use_fp8_w8a8=use_fp8_w8a8, - use_int8_w8a8=use_int8_w8a8, - use_int8_w8a16=use_int8_w8a16, - per_channel_quant=per_channel_quant, - **expand_config, - ) - - -@torch.inference_mode() -def _fused_moe_lora_fp8( - output: torch.Tensor, # (num_tokens, top_k_num, N*len(lora_a_stacked),) - qcurr_hidden_states: torch.Tensor, # (num_tokens, K,) - lora_a_stacked: list[ - torch.Tensor - ], # [(max_loras, num_experts, max_lora_rank, K,),...] - lora_b_stacked: list[ - torch.Tensor - ], # [(max_loras, num_experts, N, max_lora_rank,),...] - topk_weights: torch.Tensor, # (num_tokens, top_k_num) - sorted_token_ids: torch.Tensor | None, # (max_loras, _) - expert_ids: torch.Tensor, # (max_loras, _ ,) or (num_tokens * top_k,) - num_tokens_post_padded: torch.Tensor | None, # (max_loras, ) - token_lora_mapping: torch.Tensor, - max_lora_rank: int, - top_k_num: int, - lora_ids: torch.Tensor, - num_active_loras: int, - adapter_enabled: torch.Tensor, - shrink_block_size_m: int, - shrink_block_size_n: int, - shrink_block_size_k: int, - shrink_group_size_m: int, - shrink_num_warps: int, - shrink_num_stages: int, - shrink_split_k: int, - expand_block_size_m: int, - expand_block_size_n: int, - expand_block_size_k: int, - expand_group_size_m: int, - expand_num_warps: int, - expand_num_stages: int, - expand_split_k: int, - lora_a_scale_stacked: list[torch.Tensor], - lora_b_scale_stacked: list[torch.Tensor], - shrink_act_scale: torch.Tensor | None = None, - expand_act_scale: torch.Tensor | None = None, - mul_routed_weight: bool = False, - fully_sharded: bool = False, - offset: int = 0, - use_fp8_w8a8: bool = False, - use_int8_w8a8: bool = False, - use_int8_w8a16: bool = False, - per_channel_quant: bool = False, - block_shape: List[int] | None = None, # noqa: UP006, UP007 -) -> None: - assert len(lora_a_stacked) == len(lora_b_stacked) > 0 - assert topk_weights.dim() == qcurr_hidden_states.dim() == 2 - if sorted_token_ids is None: - assert expert_ids.dim() == 1 - else: - assert sorted_token_ids is not None - assert num_tokens_post_padded is not None - assert ( - sorted_token_ids.dim() - == expert_ids.dim() - == topk_weights.dim() - == qcurr_hidden_states.dim() - == 2 - ) - assert ( - sorted_token_ids.shape[0] - == expert_ids.shape[0] - == num_tokens_post_padded.shape[0] - ) - assert output.shape[0] == topk_weights.shape[0] - assert top_k_num == topk_weights.shape[1] - device = qcurr_hidden_states.device - num_slices = len(lora_a_stacked) - w1_lora_b_stacked = lora_b_stacked[0] - num_experts = lora_a_stacked[0].shape[1] - N = max_lora_rank - M = topk_weights.shape[0] - K = qcurr_hidden_states.shape[1] - num_tokens = M * top_k_num - w1_output_dim_size = w1_lora_b_stacked.shape[2] - assert shrink_block_size_m == expand_block_size_m - EM = ( - sorted_token_ids.shape[1] - if sorted_token_ids is not None - else num_tokens * shrink_block_size_m - ) - - a_intermediate_cache1 = torch.zeros( - (num_slices, M, top_k_num, max_lora_rank), - dtype=output.dtype, - device=device, - ) - - use_gdc = supports_pdl(device) and not fully_sharded - _fused_moe_lora_shrink_fp8( - a_intermediate_cache1, - qcurr_hidden_states, - lora_a_stacked, - topk_weights, - sorted_token_ids, - expert_ids, - num_tokens_post_padded, - token_lora_mapping, - top_k_num, - lora_ids, - adapter_enabled, - ## adding for kernel - device, - N, - M, - EM, - K, - num_tokens, - num_experts, - num_slices, - shrink_block_size_m, - shrink_block_size_n, - shrink_block_size_k, - shrink_group_size_m, - shrink_num_warps, - shrink_num_stages, - shrink_split_k, - num_active_loras, - lora_a_scale_stacked, - mul_routed_weight=mul_routed_weight, - use_gdc=use_gdc, - act_scale=shrink_act_scale, - use_fp8_w8a8=use_fp8_w8a8, - use_int8_w8a8=use_int8_w8a8, - use_int8_w8a16=use_int8_w8a16, - per_channel_quant=per_channel_quant, - block_shape=block_shape, - ) - - if fully_sharded: - if max_lora_rank == w1_lora_b_stacked.shape[-1]: - a_intermediate_cache1 = tensor_model_parallel_all_reduce( - a_intermediate_cache1 - ) - else: - a_intermediate_cache1 = tensor_model_parallel_all_gather( - a_intermediate_cache1 - ) - - # reset max_lora_rank to the full rank after allgather - max_lora_rank = a_intermediate_cache1.shape[-1] - - _fused_moe_lora_expand_fp8( - output, - a_intermediate_cache1, - lora_b_stacked, - topk_weights, - sorted_token_ids, - expert_ids, - num_tokens_post_padded, - token_lora_mapping, - top_k_num, - lora_ids, - adapter_enabled, - ## adding for kernel - device, - N, - M, - EM, - K, - num_tokens, - num_experts, - num_slices, - max_lora_rank, - w1_output_dim_size, - expand_block_size_m, - expand_block_size_n, - expand_block_size_k, - expand_group_size_m, - expand_num_warps, - expand_num_stages, - expand_split_k, - num_active_loras, - lora_b_scale_stacked, - mul_routed_weight=mul_routed_weight, - offset=offset, - use_gdc=use_gdc, - act_scale=expand_act_scale, - use_fp8_w8a8=use_fp8_w8a8, - use_int8_w8a8=use_int8_w8a8, - use_int8_w8a16=use_int8_w8a16, - per_channel_quant=per_channel_quant, - block_shape=block_shape, - ) - - -try: - direct_register_custom_op( - op_name="fused_moe_lora_fp8", - op_func=_fused_moe_lora_fp8, - mutates_args=["output"], - ) - - direct_register_custom_op( - op_name="fused_moe_lora_shrink_fp8", - op_func=_fused_moe_lora_shrink_fp8, - mutates_args=["a_intermediate_cache1"], - ) - - direct_register_custom_op( - op_name="fused_moe_lora_expand_fp8", - op_func=_fused_moe_lora_expand_fp8, - mutates_args=["output"], - ) - - fused_moe_lora_fp8 = torch.ops.vllm.fused_moe_lora_fp8 - fused_moe_lora_shrink_fp8 = torch.ops.vllm.fused_moe_lora_shrink_fp8 - fused_moe_lora_expand_fp8 = torch.ops.vllm.fused_moe_lora_expand_fp8 - -except AttributeError: - fused_moe_lora_fp8 = _fused_moe_lora_fp8 - fused_moe_lora_shrink_fp8 = _fused_moe_lora_shrink_fp8 - fused_moe_lora_expand_fp8 = _fused_moe_lora_expand_fp8 diff --git a/vllm/lora/ops/triton_ops/lora_expand_fp8_op.py b/vllm/lora/ops/triton_ops/lora_expand_fp8_op.py deleted file mode 100644 index 9369fb750b..0000000000 --- a/vllm/lora/ops/triton_ops/lora_expand_fp8_op.py +++ /dev/null @@ -1,377 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Based on: -Chen, L., Ye, Z., Wu, Y., Zhuo, D., Ceze, L., & Krishnamurthy, A. (2023). -Punica: Multi-Tenant LoRA Serving. -https://arxiv.org/abs/2310.18547 -""" - -import torch - -from vllm.lora.ops.triton_ops.fp8_kernel_utils import do_expand_kernel_fp8 -from vllm.lora.ops.triton_ops.utils import ( - _get_lora_b_ptr, - get_lora_op_configs, -) -from vllm.triton_utils import tl, triton -from vllm.utils.torch_utils import direct_register_custom_op - -_EXPAND_LORA_SCALE_PTR_DICT: dict[tuple[int, ...], torch.tensor] = {} - - -def _get_expand_lora_scale_ptr(lora_weights: list[torch.Tensor], device: torch.device): - """`_EXPAND_LORA_SCALE_PTR_DICT` collects the required information during - `profile_run`, - After this, it remains constant and subsequent usage is through LUT. - Refer to: - https://github.com/triton-lang/triton/blob/release/3.1.x/python/tutorials/08-grouped-gemm.py - """ - key = tuple(lora_weight.data_ptr() for lora_weight in lora_weights) - - if (ptr_tensor := _EXPAND_LORA_SCALE_PTR_DICT.get(key)) is not None: - return ptr_tensor - - if len(lora_weights) > 1: - tensor_ptrs = [] - for lora_weight in lora_weights: - tensor_ptrs.append(lora_weight.data_ptr()) - ptr_tensor = torch.tensor(tensor_ptrs, device=device, dtype=torch.uint64) - else: - # Single slice: return the actual tensor so the kernel can use it - # directly without pointer indirection (matches SLICE_NUM == 1 path). - ptr_tensor = lora_weights[0] - - _EXPAND_LORA_SCALE_PTR_DICT[key] = ptr_tensor - return _EXPAND_LORA_SCALE_PTR_DICT.get(key) - - -@triton.jit -def _lora_expand_kernel_fp8( - input_ptr, - lora_ptr, - out_ptr, - a_scale_ptr, - b_scale_ptr, - M, - N, - K, - token_indices_sorted_by_lora_ids, - num_tokens_per_lora, - lora_token_start_loc, - lora_ids, - slice_start_loc, - input_d0_stride, - input_d1_stride, - input_d2_stride, - ls_d0_ptr, - ls_d1_ptr, - ls_d2_ptr, - a_scale_m_stride, - a_scale_k_stride, - b_scale_l_stride, - b_scale_n_stride, - b_scale_k_stride, - output_d0_stride, - output_d1_stride, - output_hs_ptr, - group_n: tl.constexpr, - group_k: tl.constexpr, - BLOCK_M: tl.constexpr, - BLOCK_N: tl.constexpr, - BLOCK_K: tl.constexpr, - EVEN_K: tl.constexpr, - ADD_INPUTS: tl.constexpr, - CAST_TYPE: tl.constexpr, - SLICE_NUM: tl.constexpr, - SAME_STRIDE: tl.constexpr, - USE_GDC: tl.constexpr, - use_fp8_w8a8: tl.constexpr, - per_channel_quant: tl.constexpr, - launch_pdl: tl.constexpr, -): - """FP8-compatible expand kernel wrapper.""" - cta_n_num = tl.cdiv(N, BLOCK_N) - cta_m_num = tl.cdiv(M, BLOCK_M) - - pid_mn = tl.program_id(axis=0) - pid_m = pid_mn % cta_m_num - pid_n = (pid_mn // cta_m_num) % cta_n_num - - slice_id = tl.program_id(axis=1) - lora_idx = tl.program_id(axis=2) - - lora_id = tl.load(lora_ids + lora_idx) - if lora_id == -1: - return - - lora_m_size = tl.load(num_tokens_per_lora + lora_idx) - - cta_m_offset = pid_m * BLOCK_M - if cta_m_offset >= lora_m_size: - return - - curr_N = N if SAME_STRIDE else tl.load(output_hs_ptr + slice_id) - if pid_n * BLOCK_N >= curr_N: - return - - cta_m_len = min(BLOCK_M, lora_m_size - cta_m_offset) - - lora_m_indices_start = tl.load(lora_token_start_loc + lora_idx) - cta_lora_seq_indices = ( - token_indices_sorted_by_lora_ids + lora_m_indices_start + cta_m_offset - ) - - offset_m = tl.arange(0, BLOCK_M) % cta_m_len - ram = tl.load(cta_lora_seq_indices + offset_m) - - do_expand_kernel_fp8( - pid_n, - lora_id, - slice_id, - input_ptr, - lora_ptr, - out_ptr, - a_scale_ptr, - b_scale_ptr, - curr_N, - K, - cta_m_len, - ram, - slice_start_loc, - input_d0_stride, - input_d1_stride, - input_d2_stride, - ls_d0_ptr, - ls_d1_ptr, - ls_d2_ptr, - a_scale_m_stride, - a_scale_k_stride, - b_scale_l_stride, - b_scale_n_stride, - b_scale_k_stride, - output_d0_stride, - output_d1_stride, - group_n, - group_k, - BLOCK_M, - BLOCK_N, - BLOCK_K, - SAME_STRIDE, - SLICE_NUM, - EVEN_K, - CAST_TYPE, - ADD_INPUTS, - USE_GDC, - use_fp8_w8a8, - per_channel_quant, - ) - - -@torch.inference_mode() -def _lora_expand_fp8( - inputs: torch.Tensor, # shape [num_slices, num_tokens, lora_rank] - lora_b_weights: list[torch.Tensor], # FP8 [num_lora, hidden_size, lora_rank] - output_tensor: torch.Tensor, # shape [num_tokens, hidden_size * num_slices] - token_lora_mapping: torch.Tensor, - token_indices_sorted_by_lora_ids: torch.Tensor, - num_tokens_per_lora: torch.Tensor, - lora_token_start_loc: torch.Tensor, - lora_ids: torch.Tensor, - no_lora_flag_cpu: torch.Tensor, # shape [1] - num_active_loras: int, # number of active LoRAs (unused here, for API compat) - b_scale: list[torch.Tensor], # LoRA B weight scale per slice - a_scale: torch.Tensor | None = None, # Scale for shrink output (optional) - offset_start: int = 0, - add_inputs: bool = False, - group_k: int = 0, - group_n: int = 0, - use_fp8_w8a8: bool = False, - per_channel_quant: bool = False, -) -> None: - """FP8-compatible LoRA expand operation. - - Args: - inputs: Input tensor from shrink operation [num_slices, num_tokens, lora_rank] - lora_b_weights: List of FP8 LoRA B weights per slice - output_tensor: Output tensor - a_scale: Optional scale for input (if input is quantized) - b_scale: Weight quantization scales per slice - token_lora_mapping: Token to LoRA ID mapping - token_indices_sorted_by_lora_ids: Sorted token indices - num_tokens_per_lora: Number of tokens per LoRA - lora_token_start_loc: Start location for each LoRA's tokens - lora_ids: LoRA IDs to process - num_active_loras: Number of active LoRAs. Accepted for API parity with - the non-FP8 kernel and unused here. - no_lora_flag_cpu (torch.Tensor): A CPU tensor of size 1, that indicates - if there are any requests that require LoRA. - offset_start (int, optional): Offset start for output_tensor. - Defaults to 0. - add_inputs (bool, optional): Whether to add the input tensor to the - output tensor. Defaults to False. - group_k (int, optional): Block size for K in block-wise quantization. - group_n (int, optional): Block size for N in block-wise quantization. - use_fp8_w8a8 (bool, optional): Whether to use FP8 W8A8 quantization. - per_channel_quant (bool, optional): Whether to use per-channel quantization. - - """ - assert no_lora_flag_cpu.numel() == 1 - if no_lora_flag_cpu.item(): - # None of the inputs require LoRA. - return - - if use_fp8_w8a8: - assert inputs.dtype in [ - torch.float8_e4m3fn, - torch.float8_e5m2, - ] - for weight in lora_b_weights: - assert weight.dtype in [ - torch.float8_e5m2, - torch.float8_e4m3fn, - ] - else: - assert inputs.dtype in [torch.float16, torch.bfloat16, torch.float32] - for weight in lora_b_weights: - assert weight.dtype in [torch.float16, torch.bfloat16] - assert inputs.size(0) == len(lora_b_weights) - assert output_tensor.is_contiguous() - - # metadata sanity check. - M = inputs.size(1) - assert token_lora_mapping.size(0) == M - assert token_lora_mapping.size(0) == token_indices_sorted_by_lora_ids.size(0) - assert lora_ids.size(0) == num_tokens_per_lora.size(0) - assert lora_token_start_loc.size(0) == lora_ids.size(0) + 1 - - ( - slice_start_tensor, - lora_ptr_tensor, - lora_strides_d0_tensor, - lora_strides_d1_tensor, - lora_strides_d2_tensor, - hidden_sizes_tensor, - same_stride, - MAX_N, - ) = _get_lora_b_ptr(lora_b_weights, offset_start, inputs.device) - - # Get scale pointers - if b_scale is not None: - b_scale_ptr_tensor = _get_expand_lora_scale_ptr(b_scale, inputs.device) - else: - b_scale_ptr_tensor = None - K = lora_b_weights[0].shape[-1] - ADD_INPUTS = add_inputs - MAX_LORAS = lora_ids.size(0) - - CAST_TYPE = False - NUM_SLICES = len(lora_b_weights) - - # Triton kernel configs. - kernel_config = get_lora_op_configs( - op_type="expand", - max_loras=MAX_LORAS, - batch=M, - hidden_size=MAX_N, - rank=K, - num_slices=NUM_SLICES, - add_inputs=add_inputs, - ) - BLOCK_M = kernel_config["block_m"] - BLOCK_N = kernel_config["block_n"] - BLOCK_K = kernel_config["block_k"] - NUM_WARPS = kernel_config["num_warps"] - NUM_CTAS = kernel_config.get("num_ctas", 1) - NUM_STAGES = kernel_config["num_stages"] - - EVEN_K = K % BLOCK_K == 0 - - grid = ( - triton.cdiv(M, BLOCK_M) * triton.cdiv(MAX_N, BLOCK_N), - NUM_SLICES, - num_active_loras, - ) - # We disable PDL temporarily because LoRA kernels are not launching back-to-back, - # making PDL invalid and affecting the kernel performance. - use_gdc = False # supports_pdl(inputs.device) - # Get scale strides - if a_scale is not None: - a_scale_m_stride = a_scale.stride(0) if a_scale.dim() > 1 else 0 - a_scale_k_stride = a_scale.stride(-1) if a_scale.dim() > 1 else 0 - else: - a_scale_m_stride = 0 - a_scale_k_stride = 0 - - if b_scale is not None and b_scale[0].dim() > 0: - b_scale_l_stride = b_scale[0].stride(0) if b_scale[0].dim() > 0 else 0 - b_scale_n_stride = ( - b_scale[0].stride(-2) - if b_scale[0].dim() > 2 - else (b_scale[0].stride(-1) if b_scale[0].dim() > 1 else 1) - ) - b_scale_k_stride = b_scale[0].stride(-1) if b_scale[0].dim() > 2 else 0 - else: - b_scale_l_stride = 1 - b_scale_n_stride = 0 - b_scale_k_stride = 0 - - _lora_expand_kernel_fp8[grid]( - inputs, - lora_ptr_tensor, - output_tensor, - a_scale, - b_scale_ptr_tensor, - M, - MAX_N, - K, - token_indices_sorted_by_lora_ids, - num_tokens_per_lora, - lora_token_start_loc, - lora_ids, - slice_start_tensor, - inputs.stride(0), - inputs.stride(1), - inputs.stride(2), - lora_strides_d0_tensor, - lora_strides_d1_tensor, - lora_strides_d2_tensor, - a_scale_m_stride, - a_scale_k_stride, - b_scale_l_stride, - b_scale_n_stride, - b_scale_k_stride, - output_tensor.stride(0), - output_tensor.stride(1), - hidden_sizes_tensor, - group_n, - group_k, - BLOCK_M, - BLOCK_N, - BLOCK_K, - EVEN_K, - ADD_INPUTS, - CAST_TYPE, - NUM_SLICES, - same_stride, - use_gdc, - use_fp8_w8a8=use_fp8_w8a8, - per_channel_quant=per_channel_quant, - num_warps=NUM_WARPS, - num_ctas=NUM_CTAS, - num_stages=NUM_STAGES, - launch_pdl=use_gdc, - ) - - return - - -try: - direct_register_custom_op( - op_name="lora_expand_fp8", - op_func=_lora_expand_fp8, - mutates_args=["output_tensor"], - ) - lora_expand_fp8 = torch.ops.vllm.lora_expand_fp8 - -except AttributeError: - lora_expand_fp8 = _lora_expand_fp8 diff --git a/vllm/lora/ops/triton_ops/lora_shrink_fp8_op.py b/vllm/lora/ops/triton_ops/lora_shrink_fp8_op.py deleted file mode 100644 index c2b852fa40..0000000000 --- a/vllm/lora/ops/triton_ops/lora_shrink_fp8_op.py +++ /dev/null @@ -1,404 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Based on: -Chen, L., Ye, Z., Wu, Y., Zhuo, D., Ceze, L., & Krishnamurthy, A. (2023). -Punica: Multi-Tenant LoRA Serving. -https://arxiv.org/abs/2310.18547 -""" - -import torch - -from vllm.lora.ops.triton_ops.fp8_kernel_utils import do_shrink_kernel_fp8 -from vllm.lora.ops.triton_ops.utils import _get_lora_a_ptr, get_lora_op_configs -from vllm.triton_utils import tl, triton -from vllm.utils.torch_utils import direct_register_custom_op - -_SHRINK_LORA_SCALE_PTR_DICT: dict[tuple[int, ...], tuple] = {} - - -def _get_shrink_lora_scale_ptr( - lora_scale_weights: list[torch.Tensor], device: torch.device -): - """`_SHRINK_LORA_SCALE_PTR_DICT` collects the required information during - `profile_run`. After this, it remains constant and subsequent usage is - through LUT. - - Returns a tuple of (scale_ptr_tensor, l_stride, n_stride, k_stride). - - Supports scale tensors of varying dimensionality: - - 1D: (lora_num,) — tensor-wise quantization - - 2D: (lora_num, N) — per-channel quantization - - 3D: (lora_num, N, K) — block-wise quantization - - 4D: (lora_num, 1, N, K) — block-wise with extra dim (squeezed to 3D) - - Refer to: - https://github.com/triton-lang/triton/blob/release/3.1.x/python/tutorials/08-grouped-gemm.py - """ - key = tuple(lora_weight.data_ptr() for lora_weight in lora_scale_weights) - - if values := _SHRINK_LORA_SCALE_PTR_DICT.get(key): - return values - - tensor_ptrs = [] - scale_l_strides = [] - scale_n_strides = [] - scale_k_strides = [] - for lora_scale_weight in lora_scale_weights: - if lora_scale_weight.ndim == 4: # shape:(lora_num,1,size,rank) - assert lora_scale_weight.size(1) == 1 - lora_scale_weight = lora_scale_weight.squeeze(dim=1) - assert 1 <= lora_scale_weight.ndim <= 3 - assert lora_scale_weight.is_contiguous() - tensor_ptrs.append(lora_scale_weight.data_ptr()) - scale_l_strides.append( - lora_scale_weight.stride(0) if lora_scale_weight.ndim > 0 else 0 - ) - scale_n_strides.append( - lora_scale_weight.stride(-2) - if lora_scale_weight.ndim > 2 - else (lora_scale_weight.stride(-1) if lora_scale_weight.ndim > 1 else 1) - ) - scale_k_strides.append( - lora_scale_weight.stride(-1) if lora_scale_weight.ndim > 2 else 0 - ) - if len(lora_scale_weights) > 1: - scale_ptr_tensor = torch.tensor(tensor_ptrs, device=device, dtype=torch.uint64) - else: - scale_ptr_tensor = lora_scale_weights[0] - - if ( - len(set(scale_l_strides)) > 1 - or len(set(scale_n_strides)) > 1 - or len(set(scale_k_strides)) > 1 - ): - raise ValueError("All LoRA scale weights must have the same stride.") - - _SHRINK_LORA_SCALE_PTR_DICT[key] = ( - scale_ptr_tensor, - scale_l_strides[0], - scale_n_strides[0], - scale_k_strides[0], - ) - return _SHRINK_LORA_SCALE_PTR_DICT.get(key) - - -@triton.jit -def _lora_shrink_kernel_fp8( - input_ptr, - lora_ptr, - out_ptr, - a_scale_ptr, - b_scale_ptr, - M, - N, - K, - token_indices_sorted_by_lora_ids, - num_tokens_per_lora, - lora_token_start_loc, - lora_ids, - scaling, - input_d0_stride, - input_d1_stride, - lora_d0_stride, - lora_d1_stride, - lora_d2_stride, - a_scale_m_stride, - a_scale_k_stride, - b_scale_l_stride, - b_scale_n_stride, - b_scale_k_stride, - output_d0_stride, - output_d1_stride, - output_d2_stride, - group_n: tl.constexpr, - group_k: tl.constexpr, - BLOCK_M: tl.constexpr, - BLOCK_N: tl.constexpr, - BLOCK_K: tl.constexpr, - EVEN_K: tl.constexpr, - SPLIT_K: tl.constexpr, - GROUP_SIZE_M: tl.constexpr, - SLICE_NUM: tl.constexpr, - USE_GDC: tl.constexpr, ## should always be false in shrink kernel - use_fp8_w8a8: tl.constexpr, - per_channel_quant: tl.constexpr, - launch_pdl: tl.constexpr, -): - cta_n_num = tl.cdiv(N, BLOCK_N) - cta_m_num = tl.cdiv(M, BLOCK_M) - - pid_sk_m_n = tl.program_id(axis=0) - pid_sk = pid_sk_m_n % SPLIT_K - - pid_m_n = pid_sk_m_n // SPLIT_K - num_pid_in_group = GROUP_SIZE_M * cta_n_num - group_id = pid_m_n // num_pid_in_group - first_pid_m = group_id * GROUP_SIZE_M - - group_size_m = min(cta_m_num - first_pid_m, GROUP_SIZE_M) - - # Column-major ordering within groups for better cache reuse - pid_m = first_pid_m + ((pid_m_n % num_pid_in_group) % group_size_m) - pid_n = (pid_m_n % num_pid_in_group) // group_size_m - - slice_id = tl.program_id(axis=1) - lora_idx = tl.program_id(axis=2) - - lora_id = tl.load(lora_ids + lora_idx) - if lora_id == -1: - # Early exit for the no-lora case. - return - - lora_m_size = tl.load(num_tokens_per_lora + lora_idx) - - cta_m_offset = pid_m * BLOCK_M - if cta_m_offset >= lora_m_size: - # Early exit CTA. - return - - # num rows this CTA should process. - cta_m_len = min(BLOCK_M, lora_m_size - cta_m_offset) - - # Identify all rows that this CTA should process. - lora_m_indices_start = tl.load(lora_token_start_loc + lora_idx) - cta_lora_seq_indices = ( - token_indices_sorted_by_lora_ids + lora_m_indices_start + cta_m_offset - ) - - # Load all relevant row indices. - offset_m = tl.arange(0, BLOCK_M) % cta_m_len - ram = tl.load(cta_lora_seq_indices + offset_m) - - do_shrink_kernel_fp8( - pid_n, - pid_sk, - slice_id, - lora_id, - input_ptr, - lora_ptr, - out_ptr, - a_scale_ptr, - b_scale_ptr, - N, - K, - cta_m_len, - ram, # array identifying the rows of Input ptr to operate on - # input strides - input_d0_stride, - input_d1_stride, - # lora strides - lora_d0_stride, - lora_d1_stride, - lora_d2_stride, - # scale strides - a_scale_m_stride, - a_scale_k_stride, - b_scale_l_stride, - b_scale_n_stride, - b_scale_k_stride, - # output strides - output_d0_stride, - output_d1_stride, - output_d2_stride, - scaling, - # block size for block-wise quantization - group_n, - group_k, - BLOCK_M, - BLOCK_N, - BLOCK_K, - EVEN_K, - SPLIT_K, - SLICE_NUM, - USE_GDC, - use_fp8_w8a8, - per_channel_quant, - launch_pdl, - ) - - -@torch.inference_mode() -def _lora_shrink_fp8( - inputs: torch.Tensor, # shape [num_tokens, hidden_size] - FP8 or FP16/BF16 - lora_a_weights: list[ - torch.Tensor - ], # shape [num_loras, lora_rank, hidden_size] - FP8 or FP16/BF16 - output_tensor: torch.Tensor, # shape [num_slices, num_tokens, lora_rank] - token_lora_mapping: torch.Tensor, # shape [num_tokens] - token_indices_sorted_by_lora_ids: torch.Tensor, # shape [num_tokens] - num_tokens_per_lora: torch.Tensor, # shape [max-loras + 1] - lora_token_start_loc: torch.Tensor, # shape [max-loras + 2] - lora_ids: torch.Tensor, # shape [max-loras + 1] - no_lora_flag_cpu: torch.Tensor, # shape [1] - num_active_loras: int, # number of active LoRAs (unused here, for API compat) - scaling: float, - b_scale: list[torch.Tensor], # LoRA weight scale per slice - a_scale: torch.Tensor | None = None, # Activation scale - per-token or block-wise - group_k: int = 0, # Block size for K in block-wise quantization (0 = tensor-wise) - group_n: int = 0, # Block size for N in block-wise quantization - use_fp8_w8a8: bool = False, - per_channel_quant: bool = False, -) -> None: - """Args: - inputs: FP8 or FP16/BF16 input tensor [num_tokens, hidden_size] - lora_a_weights: List of FP8 or FP16/BF16 LoRA A weights per slice - output_tensor: Output tensor (FP16/BF16/FP32) - token_lora_mapping: Token to LoRA ID mapping - token_indices_sorted_by_lora_ids: Sorted token indices - num_tokens_per_lora: Number of tokens per LoRA - lora_token_start_loc: Start location for each LoRA's tokens - lora_ids: LoRA IDs to process - scaling: LoRA scaling factor - a_scale: Activation quantization scales - b_scale: Weight quantization scales per slice - group_k: Block size for K dimension quantization - group_n: Block size for N dimension quantization - use_fp8_w8a8: Whether to use FP8 weights and activations - per_channel_quant: Whether to use per-channel quantization - - """ - assert no_lora_flag_cpu.numel() == 1 - if no_lora_flag_cpu.item(): - # None of the inputs require LoRA. - return - - assert inputs.size(1) == lora_a_weights[0].size(-1) - assert inputs.is_contiguous() - assert output_tensor.is_contiguous() - - # metadata sanity check - M = inputs.size(0) - assert token_lora_mapping.size(0) == M - assert token_lora_mapping.size(0) == token_indices_sorted_by_lora_ids.size(0) - assert lora_ids.size(0) == num_tokens_per_lora.size(0) - assert lora_token_start_loc.size(0) == lora_ids.size(0) + 1 - - output_tensor.zero_() - - # Get LoRA weight pointers - (lora_ptr_tensor, lora_strides_d0, lora_strides_d1, lora_strides_d2) = ( - _get_lora_a_ptr(lora_a_weights, inputs.device) - ) - - # Get scale pointers if using FP8 - if use_fp8_w8a8: - assert a_scale is not None, "a_scale required for FP8 w8a8" - assert b_scale is not None, "b_scale required for FP8" - - b_scale_ptr_tensor, b_scale_l_stride, b_scale_n_stride, b_scale_k_stride = ( - _get_shrink_lora_scale_ptr(b_scale, inputs.device) - ) - a_scale_ptr = ( - a_scale if a_scale is not None else torch.tensor(1.0, device=inputs.device) - ) - else: - b_scale_ptr_tensor = torch.tensor(0, device=inputs.device) - b_scale_l_stride = 0 - b_scale_n_stride = 0 - b_scale_k_stride = 0 - a_scale_ptr = torch.tensor(0, device=inputs.device) - - N, K = lora_a_weights[0].shape[-2:] # K=hidden_size, N=rank - NUM_SLICES = len(lora_a_weights) - MAX_LORAS = lora_ids.size(0) - - # Triton kernel configs - kernel_config = get_lora_op_configs( - "shrink", - max_loras=MAX_LORAS, - batch=M, - hidden_size=K, - rank=N, - num_slices=NUM_SLICES, - ) - BLOCK_M = kernel_config["block_m"] - BLOCK_N = kernel_config["block_n"] - BLOCK_K = kernel_config["block_k"] - SPLIT_K = kernel_config["split_k"] - NUM_WARPS = kernel_config["num_warps"] - NUM_STAGES = kernel_config["num_stages"] - NUM_CTAS = kernel_config["num_ctas"] - GROUP_SIZE_M = kernel_config.get("group_size_m", 8) - assert BLOCK_K is not None and SPLIT_K is not None - EVEN_K = K % (BLOCK_K * SPLIT_K) == 0 - - # Grid configuration with column-major ordering support - grid = ( - SPLIT_K * triton.cdiv(M, BLOCK_M) * triton.cdiv(N, BLOCK_N), - NUM_SLICES, - num_active_loras, - ) - - # Determine scale strides - if use_fp8_w8a8: - if a_scale is not None and a_scale.ndim == 2: - a_scale_m_stride = a_scale.stride(0) - a_scale_k_stride = a_scale.stride(1) - else: - a_scale_m_stride = 0 - a_scale_k_stride = 0 - else: - a_scale_m_stride = 0 - a_scale_k_stride = 0 - - # We disable PDL temporarily because LoRA kernels are not launching back-to-back, - # making PDL invalid and affecting the kernel performance. - use_gdc = False # supports_pdl(inputs.device) - _lora_shrink_kernel_fp8[grid]( - inputs, - lora_ptr_tensor, - output_tensor, - a_scale_ptr, - b_scale_ptr_tensor, - M, - N, - K, - token_indices_sorted_by_lora_ids, - num_tokens_per_lora, - lora_token_start_loc, - lora_ids, - scaling, - inputs.stride(0), - inputs.stride(1), - lora_strides_d0, - lora_strides_d1, - lora_strides_d2, - a_scale_m_stride, - a_scale_k_stride, - b_scale_l_stride, - b_scale_n_stride, - b_scale_k_stride, - output_tensor.stride(0), - output_tensor.stride(1), - output_tensor.stride(2), - group_n, - group_k, - BLOCK_M, - BLOCK_N, - BLOCK_K, - EVEN_K, - SPLIT_K, - GROUP_SIZE_M, - NUM_SLICES, - use_gdc, - use_fp8_w8a8, - per_channel_quant, - use_gdc, - num_warps=NUM_WARPS, - num_ctas=NUM_CTAS, - num_stages=NUM_STAGES, - ) - - return - - -try: - direct_register_custom_op( - op_name="lora_shrink_fp8", - op_func=_lora_shrink_fp8, - mutates_args=["output_tensor"], - ) - lora_shrink_fp8 = torch.ops.vllm.lora_shrink_fp8 - -except AttributeError: - lora_shrink_fp8 = _lora_shrink_fp8 diff --git a/vllm/lora/punica_wrapper/punica_cpu.py b/vllm/lora/punica_wrapper/punica_cpu.py deleted file mode 100644 index 436445554f..0000000000 --- a/vllm/lora/punica_wrapper/punica_cpu.py +++ /dev/null @@ -1,371 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from collections.abc import Callable -from typing import Any - -import torch - -from vllm.lora.ops.torch_ops import ( - bgmv_expand, - bgmv_expand_slice, - bgmv_shrink, - sgmv_expand, - sgmv_expand_slice, - sgmv_shrink, -) - -from .punica_base import PunicaWrapperBase - - -# The platforms that are compatible with the PyTorch-native implementation can -# inherit this class -class PunicaWrapperCPU(PunicaWrapperBase): - """PunicaWrapperCPU is designed to manage and provide metadata for the punica - kernel. The main function is to maintain the state information for - Multi-LoRA, and to provide the interface for the pytorch punica ops. - """ - - def __init__( - self, - max_num_batched_tokens: int, - max_batches: int, - device: torch.device | str, - **kwargs, - ): - PunicaWrapperBase.__init__(self, max_num_batched_tokens, max_batches, device) - - def _shrink_prefill( - self, - y: torch.Tensor, - x: torch.Tensor, - w_t_all: torch.Tensor, - scale: float, - ): - # No LoRA request, so return directly - if self.no_lora: - return - sgmv_shrink( - x, - w_t_all, - y, - *self.prefill_metadata, - scale, - ) - - def _shrink_decode( - self, - y: torch.Tensor, - x: torch.Tensor, - w_t_all: torch.Tensor, - scale: float, - ): - bgmv_shrink(x, w_t_all, y, self.token_lora_indices, scale) - - def _expand_prefill( - self, - y: torch.Tensor, - x: torch.Tensor, - w_t_all: torch.Tensor, - add_inputs: bool, - ): - # No LoRA request, so return directly - if self.no_lora: - return - sgmv_expand( - x, - w_t_all, - y, - *self.prefill_metadata, - add_inputs, - ) - - def _expand_decode( - self, - y: torch.Tensor, - x: torch.Tensor, - w_t_all: torch.Tensor, - add_inputs: bool, - ): - bgmv_expand(x, w_t_all, y, self.token_lora_indices, add_inputs) - - def _expand_slice_prefill( - self, - y: torch.Tensor, - x: torch.Tensor, - w_t_all: torch.Tensor, - y_offset: int, - y_slice_size: int, - add_inputs: bool, - ): - # No LoRA request, so return directly - if self.no_lora: - return - sgmv_expand_slice( - x, - w_t_all, - y, - *self.prefill_metadata, - y_offset, - y_slice_size, - add_inputs, - ) - - def _expand_slice_decode( - self, - y: torch.Tensor, - x: torch.Tensor, - w_t_all: torch.Tensor, - y_offset: int, - y_slice_size: int, - add_inputs: bool, - ): - bgmv_expand_slice( - x, w_t_all, y, self.token_lora_indices, y_offset, y_slice_size, add_inputs - ) - - def _apply_expand( - self, - y: torch.Tensor, - x: torch.Tensor, - w_t_all: torch.Tensor, - y_offset: int, - y_slice_size: int, - add_inputs: bool = True, - ): - """Perform the ` y[:,y_offset:y_offset+y_slice_size]+=x@w_t_all` - computation, which is suitable for the - GEMM of lora'b. - """ - expand_slice_fun: Callable = ( - self._expand_slice_prefill if self.is_prefill else self._expand_slice_decode - ) - expand_slice_fun(y, x, w_t_all, y_offset, y_slice_size, add_inputs) - - def _apply_shrink( - self, y: torch.Tensor, x: torch.Tensor, w_t_all: torch.Tensor, scale: float - ): - """Perform the ` y+=x@w_t_all` computation, which is suitable for the - GEMM of lora'a. - When `is_prefill is` true, it indicates that it is currently the - prefill stage, and the `_shrink_prefill` function should be called. - Otherwise, it is the decode stage, and the _shrink_decode function - should be called. - """ - y_org = y - y = y.view(-1, y.shape[-1]) - shrink_fun: Callable = ( - self._shrink_prefill if self.is_prefill else self._shrink_decode - ) - shrink_fun(y, x, w_t_all, scale) - y = y.view_as(y_org) - - def add_shrink( - self, - y: tuple[torch.Tensor, ...] | torch.Tensor, - x: torch.Tensor, - lora_a_stacked: tuple[torch.Tensor, ...], - scale: float, - **kwargs: Any, - ): - """Performs GEMM for multiple slices of lora_a. - When `is_prefill is` true, it indicates that it is currently the - prefill stage, and the `_shrink_prefill` function should be called. - Otherwise, it is the decode stage, and the _shrink_decode function - should be called. - - Semantics: - for i in range(len(lora_a_stacked)): - y[i] += (x @ lora_a_stacked[i]) * scale - - Args: - y (Union[tuple[torch.Tensor, ...], torch.Tensor]): Output tensors - x (torch.Tensor): Input tensor - lora_a_stacked (tuple[torch.Tensor, ...]): lora_a's weights - scale (float): Scaling factor for the operation - **kwargs: Unused; accepted for compatibility with the base class signature. - - """ - x = x.view(-1, x.shape[-1]) - # TODO fuse these kernels - for slice_idx in range(len(lora_a_stacked)): - self._apply_shrink(y[slice_idx], x, lora_a_stacked[slice_idx], scale) - - def add_expand( - self, - y: torch.Tensor, - x: tuple[torch.Tensor, ...] | torch.Tensor, - lora_b_stacked: tuple[torch.Tensor, ...], - output_slices: tuple[int, ...], - offset_start: int = 0, - add_inputs=True, - **kwargs: Any, - ) -> None: - """Performs GEMM for multiple slices of lora_b. - - Semantics: - for i in range(len(lora_b_stacked)): - slice = output_slices[i] - y[:, offset:offset+slice] += x[i] @ lora_b_stacked[i] - offset += slice - - Args: - y (torch.Tensor): Output tensor. - x (Union[tuple[torch.Tensor, ...], torch.Tensor]): Input tensors - lora_b_stacked (tuple[torch.Tensor, ...]): lora_b's weight - output_slices (tuple[int, ...]): Every slice's size - offset_start (int): The starting position of y, defaults to 0. - add_inputs (bool): Defaults to True. - **kwargs: Unused; accepted for compatibility with the base class signature. - - """ - y_org = y - y = y.view(-1, y.shape[-1]) - offset_left = offset_start - for slice_idx in range(len(lora_b_stacked)): - self._apply_expand( - y, - x[slice_idx], - lora_b_stacked[slice_idx], - offset_left, - output_slices[slice_idx], - add_inputs=add_inputs, - ) - offset_left += output_slices[slice_idx] - y = y.view_as(y_org) - - def add_lora_embedding( - self, - y: torch.Tensor, - x: torch.Tensor, - lora_b_stacked: torch.Tensor, - add_inputs: bool = True, - **kwargs: Any, - ) -> None: - """Applies lora specifically for VocabParallelEmbeddingWithLoRA. - - Semantics: - y += x @ lora_b_stacked - - Args: - y (torch.Tensor): Output tensor. - x (torch.Tensor): Input tensor. - lora_b_stacked (torch.Tensor): lora_b's weights. - add_inputs (bool): Default to True. - **kwargs: Unused; accepted for compatibility with the base class signature. - - """ - # Embedding layer only need expand op - expand_fun: Callable = ( - self._expand_prefill if self.is_prefill else self._expand_decode - ) - expand_fun(y, x, lora_b_stacked, add_inputs) - - def add_lora_linear( - self, - y: torch.Tensor, - x: torch.Tensor, - lora_a_stacked: tuple[torch.Tensor, ...], - lora_b_stacked: tuple[torch.Tensor, ...], - scale: float, - output_slices: tuple[int, ...], - *, - buffer: tuple[torch.Tensor, ...] | None = None, - **kwargs: Any, - ) -> None: - """Applicable to linear-related lora. - - Semantics: - for i in range(len(lora_a_stacked)): - y[i] += ( - x[i].unsqueeze(0) - @ lora_a_stacked[indices[i], layer_idx, :, :] - @ lora_b_stacked[indices[i], layer_idx, :, :] - * scale - ).squeeze(0) - - Args: - y (torch.Tensor): Output tensor. Will be changed in-place. - x (torch.Tensor): Input tensor - lora_a_stacked (tuple[torch.Tensor, ...]): lora_a's weight. - lora_b_stacked (tuple[torch.Tensor, ...]): lora_b's weight. - scale (float): Scaling factor. - output_slices (tuple[int, ...]): Every slice's size. - buffer (Optional[tuple[torch.Tensor, ...]]): Defaults to None. - **kwargs: Unused; accepted for compatibility with the base class signature. - - """ - assert len(lora_a_stacked) == len(lora_b_stacked) == len(output_slices) - - if buffer is None: - r = lora_b_stacked[0].size(-1) - # We set the buffer to be float32 by default, consistent with the - # triton op - buffer = tuple( - torch.zeros((x.size(0), r), dtype=torch.float32, device=x.device) - for _ in range(len(output_slices)) - ) - self.add_shrink(buffer, x, lora_a_stacked, scale, **kwargs) - self.add_expand( - y, buffer, lora_b_stacked, output_slices, add_inputs=True, **kwargs - ) - - def add_lora_logits( - self, - y: torch.Tensor, - x: torch.Tensor, - lora_a_stacked: torch.Tensor, - lora_b_stacked: torch.Tensor, - scale, - *, - buffer: torch.Tensor | None = None, - **kwargs: Any, - ) -> None: - """Applies lora specifically for LogitsProcessorWithLoRA. - - Semantics: - buffer = (x @ lora_a_stacked) * scale - y += buffer @ lora_b_stacked - - Args: - y (torch.Tensor): Output tensor. - x (torch.Tensor): Input tensor. - lora_a_stacked (torch.Tensor): lora_a's weights. - lora_b_stacked (torch.Tensor):lora_b's weights. - scale (float): Scaling factor. - buffer (Optional[torch.Tensor]):Default to None. - **kwargs: Forwarded to `add_shrink` and `add_expand`. - - """ - y_org = y - y = y.view(-1, y.shape[-1]) - x = x.view(-1, x.shape[-1]) - r = lora_b_stacked.size(-1) - if buffer is None: - # We set the buffer to be float32 by default, consistent with the - # triton op - buffer = torch.zeros((x.size(0), r), dtype=torch.float32, device=x.device) - # LogitsProcessorWithLoRA always using bgmv. - bgmv_shrink(x, lora_a_stacked, buffer, self.sampler_indices, scale) - bgmv_expand(buffer, lora_b_stacked, y, self.sampler_indices, add_inputs=True) - y = y.view_as(y_org) - - def apply_lora_full_linear( - self, - y: torch.Tensor, - x: torch.Tensor, - weight_stacked: torch.Tensor, - bias_stacked: torch.Tensor, - module_enabled: torch.Tensor, - ) -> None: - indices = self.sampler_indices - adapter_y = torch.zeros( - (x.size(0), weight_stacked.size(-2)), - dtype=torch.float32, - device=x.device, - ) - bgmv_shrink(x, weight_stacked, adapter_y, indices, 1.0) - result = self._select_full_linear_output( - y, adapter_y, bias_stacked, module_enabled - ) - y.copy_(result) diff --git a/vllm/model_executor/kernels/attention/dsa/__init__.py b/vllm/model_executor/kernels/attention/dsa/__init__.py deleted file mode 100644 index 208f01a7cb..0000000000 --- a/vllm/model_executor/kernels/attention/dsa/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project diff --git a/vllm/model_executor/kernels/attention/dsa/candidate_blocks.py b/vllm/model_executor/kernels/attention/dsa/candidate_blocks.py deleted file mode 100644 index a822b0ff0b..0000000000 --- a/vllm/model_executor/kernels/attention/dsa/candidate_blocks.py +++ /dev/null @@ -1,271 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -import torch - -from vllm.model_executor.kernels.attention.dsa.sparse_mqa_logits import has_deep_select -from vllm.model_executor.layers.indexer_topk import ( - deep_select_topk, - get_deep_select_stride_requirement, - is_deep_select_supported, -) -from vllm.triton_utils import tl, triton -from vllm.utils.math_utils import round_up - - -@triton.jit -def _max_with_nan(a, b): - return tl.maximum(a, b, propagate_nan=tl.PropagateNan.ALL) - - -@triton.jit(do_not_specialize=["width", "nblocks"]) -def _block_scores_kernel( - logits, - starts, - ends, - scores, - valid_blocks, - stride_row, - stride_col, - stride_start, - stride_end, - stride_scores, - width, - nblocks, - BLOCK_SIZE: tl.constexpr, - HAS_STARTS: tl.constexpr, - ROW_REPEAT: tl.constexpr, - TILE: tl.constexpr, - FULL_WIDTH: tl.constexpr, -): - # Writes each row's valid block count; FULL_WIDTH also scores the -inf tail. - row = tl.program_id(0).to(tl.int64) - start = tl.load(starts + row // ROW_REPEAT * stride_start) if HAS_STARTS else 0 - end = tl.load(ends + row // ROW_REPEAT * stride_end) - col_end = tl.minimum(end, width) - num_valid = tl.cdiv(tl.maximum(col_end - start, 0), BLOCK_SIZE) - if tl.program_id(1) == 0: - tl.store(valid_blocks + row, num_valid) - num_blocks = nblocks if FULL_WIDTH else num_valid - offsets = tl.arange(0, triton.next_power_of_2(BLOCK_SIZE)) - num_tiles = tl.cdiv(num_blocks, TILE) - for tile_id in range(tl.program_id(1), num_tiles, tl.num_programs(1)): - blocks = tile_id * TILE + tl.arange(0, TILE) - cols = start + blocks[:, None] * BLOCK_SIZE + offsets[None, :] - values = tl.load( - logits + row * stride_row + cols * stride_col, - (offsets[None, :] < BLOCK_SIZE) & (cols < col_end), - other=-float("inf"), - ).to(tl.float32) - reduced = tl.reduce(values, 1, _max_with_nan) - # Pin the newest block. DeepSelect leaves NaN rows partially written, - # so also pin NaN blocks (torch.topk ranks NaN first too). - pinned = (end > start) & (blocks == (end - start - 1) // BLOCK_SIZE) - if not FULL_WIDTH: - pinned |= reduced != reduced - reduced = tl.where(pinned, float("inf"), reduced) - tl.store(scores + row * stride_scores + blocks, reduced, blocks < num_blocks) - - -@triton.jit(do_not_specialize=["k"]) -def _store_candidates_kernel( - values, - indices, - output, - out_stride_row, - out_stride_col, - k, - OUT_K: tl.constexpr, - TILE: tl.constexpr, -): - row = tl.program_id(0).to(tl.int64) - cols = tl.program_id(1) * TILE + tl.arange(0, TILE) - value = tl.load(values + row * k + cols, cols < k, other=-float("inf")) - index = tl.load(indices + row * k + cols, cols < k, other=-1) - # NaN scores can occur during warmup; only -inf denotes padding. - tl.store( - output + row * out_stride_row + cols * out_stride_col, - tl.where(value != -float("inf"), index, -1), - cols < OUT_K, - ) - - -@triton.jit(do_not_specialize=["width", "nblocks"]) -def _candidate_flags_kernel( - candidates, - starts, - flags, - stride_row, - stride_col, - stride_start, - width, - nblocks, - BLOCK_SIZE: tl.constexpr, - K: tl.constexpr, - HAS_STARTS: tl.constexpr, - ROW_REPEAT: tl.constexpr, -): - row = tl.program_id(0).to(tl.int64) - offsets = tl.arange(0, 1024) - for tile in range(tl.cdiv(nblocks + 1, 1024)): - slots = tile * 1024 + offsets - tl.store(flags + row * (nblocks + 1) + slots, 0, slots <= nblocks) - start = tl.load(starts + row // ROW_REPEAT * stride_start) if HAS_STARTS else 0 - cols = tl.arange(0, triton.next_power_of_2(K)) - block = tl.load( - candidates + row * stride_row + cols * stride_col, cols < K, other=-1 - ).to(tl.int64) - # Preserve the packed-column clamp for candidates beyond the logits width. - block = tl.where(start + block * BLOCK_SIZE >= width, nblocks, block) - tl.debug_barrier() - tl.store(flags + row * (nblocks + 1) + block, 1, (cols < K) & (block >= 0)) - - -@triton.jit(do_not_specialize=["width", "nblocks"]) -def _mask_candidates_kernel( - logits, - starts, - ends, - flags, - stride_row, - stride_col, - stride_start, - stride_end, - width, - nblocks, - BLOCK_SIZE: tl.constexpr, - HAS_STARTS: tl.constexpr, - ROW_REPEAT: tl.constexpr, - TILE: tl.constexpr, -): - row = tl.program_id(0).to(tl.int64) - cols = tl.program_id(1) * TILE + tl.arange(0, TILE) - start = tl.load(starts + row // ROW_REPEAT * stride_start) if HAS_STARTS else 0 - end = tl.load(ends + row // ROW_REPEAT * stride_end) - valid = (cols >= start) & (cols < end) & (cols < width) - block = (cols - start) // BLOCK_SIZE - keep = tl.load(flags + row * (nblocks + 1) + block, valid, other=0) - edge = tl.load(flags + row * (nblocks + 1) + nblocks) - keep = (keep != 0) | ((cols == width - 1) & (edge != 0)) - tl.store( - logits + row * stride_row + cols * stride_col, - -float("inf"), - (cols < width) & ~(valid & keep), - ) - - -def select_candidate_blocks( - logits: torch.Tensor, - row_ks: torch.Tensor | None, - row_ke: torch.Tensor, - topk_blocks: int, - block_size: int, - out: torch.Tensor, - row_repeat: int = 1, -) -> None: - """Select local block IDs by maximum score, pinning each row's newest block. - - Row bounds are in packed column space; absent starts mean zero. - Decode rows share bounds in groups of ``row_repeat``. Output is -1 padded. - """ - assert logits.is_cuda - rows, width = logits.shape - if not rows: - return - if not width: - out.fill_(-1) - return - nblocks = triton.cdiv(width, block_size) - # Logits are max_model_len wide (CUDA-graph static) but rows are shorter: - # DeepSelect stops at each row's valid block count. Order does not matter. - use_deep_select = has_deep_select() and nblocks >= topk_blocks - if use_deep_select: - in_align, out_align = get_deep_select_stride_requirement() - padded = round_up(nblocks, in_align // logits.element_size()) - scores = logits.new_empty((rows, padded))[:, :nblocks] - use_deep_select = ( - out.stride(1) == 1 - and out.stride(0) * out.element_size() % out_align == 0 - and is_deep_select_supported(scores, topk_blocks) - ) - else: - scores = logits.new_empty((rows, nblocks)) - valid_blocks = torch.empty(rows, dtype=torch.int32, device=logits.device) - _block_scores_kernel[(rows, 32)]( - logits, - row_ks, - row_ke, - scores, - valid_blocks, - *logits.stride(), - row_ks.stride(0) if row_ks is not None else 0, - row_ke.stride(0), - scores.stride(0), - width, - nblocks, - block_size, - row_ks is not None, - row_repeat, - TILE=128, - FULL_WIDTH=not use_deep_select, - ) - if use_deep_select: - deep_select_topk(scores, topk_blocks, end=valid_blocks, output_idx=out) - return - # Keep the existing top-k tie behavior. - top = scores.topk(min(topk_blocks, nblocks), dim=-1) - _store_candidates_kernel[(rows, triton.cdiv(topk_blocks, 256))]( - top.values, - top.indices, - out, - *out.stride(), - top.values.shape[1], - topk_blocks, - 256, - ) - - -def apply_candidate_mask( - logits: torch.Tensor, - row_ks: torch.Tensor | None, - row_ke: torch.Tensor, - candidate_blocks: torch.Tensor, - block_size: int, - row_repeat: int = 1, -) -> None: - """Mask packed logits outside causal bounds and request-local candidates.""" - assert logits.is_cuda - rows, width = logits.shape - if not rows or not width: - return - nblocks = triton.cdiv(width, block_size) - flags = torch.empty((rows, nblocks + 1), device=logits.device, dtype=torch.uint8) - start_stride = row_ks.stride(0) if row_ks is not None else 0 - _candidate_flags_kernel[(rows,)]( - candidate_blocks, - row_ks, - flags, - *candidate_blocks.stride(), - start_stride, - width, - nblocks, - block_size, - candidate_blocks.shape[1], - row_ks is not None, - row_repeat, - ) - _mask_candidates_kernel[(rows, triton.cdiv(width, 1024))]( - logits, - row_ks, - row_ke, - flags, - *logits.stride(), - start_stride, - row_ke.stride(0), - width, - nblocks, - block_size, - row_ks is not None, - row_repeat, - 1024, - ) diff --git a/vllm/model_executor/kernels/attention/dsa/sparse_mqa_logits.py b/vllm/model_executor/kernels/attention/dsa/sparse_mqa_logits.py deleted file mode 100644 index 0e3ae398f6..0000000000 --- a/vllm/model_executor/kernels/attention/dsa/sparse_mqa_logits.py +++ /dev/null @@ -1,498 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""DeepGEMM sparse MQA-logits path for the DeepSeek V4.1 two-level indexer. - -The candidate-source indexer publishes the top candidate blocks (in units of -``candidate_block_size`` compressed positions). Instead of scoring every KV -position and masking the dense logits down to those blocks, this path scores -only the candidate tokens with DeepGEMM's sparse MQA logits kernels -(``fp8_fp4_(paged_)sparse_mqa_logits``, DeepGEMM >= 2.8, SM100 only), runs -DeepSelect's top-k directly on the bf16 logits they produce, and remaps the -selected columns back to request-local positions. - -Only the MXFP4 indexer cache is supported: the kernels require UE8M0-packed -(granularity-32) Q/KV scales, which is exactly the MXFP4 cache layout. The -FP8 indexer cache stores one fp32 scale per quant block instead. - -The functions here take caller-owned output buffers (``sparse_indices``, -``end``, ``col_indices``) so the hot path allocates nothing; the indexer -metadata builder owns them. -""" - -import importlib.util - -import torch - -from vllm.model_executor.layers.indexer_topk import ( - deep_select_topk, - get_deep_select_stride_requirement, -) -from vllm.platforms import current_platform -from vllm.triton_utils import tl, triton -from vllm.utils.deep_gemm import ( - fp8_fp4_paged_sparse_mqa_logits, - fp8_fp4_sparse_mqa_logits, - get_paged_sparse_mqa_logits_metadata, - get_sparse_mqa_logits_metadata, -) - -# Sparse-block sizes (in tokens) supported by the DeepGEMM kernels. -SPARSE_BLOCK_KV_CHOICES = (16, 8) - - -def has_deep_select() -> bool: - """Whether the DeepSelect top-k extension is built and this GPU runs it.""" - return ( - importlib.util.find_spec("vllm._deepselect_C") is not None - and current_platform.is_cuda() - and current_platform.is_device_capability_family(100) - ) - - -def check_deep_select_layout(num_sparse_cols: int, topk_tokens: int) -> None: - """Raise unless DeepSelect accepts the sparse logits layout. - - The sparse kernels return contiguous bf16 ``[rows, num_sparse_cols]`` - logits and the top-k writes int32 ``[rows, topk_tokens]`` indices; - DeepSelect needs both row strides aligned (1024B / 32B on SM100). - """ - in_align, out_align = get_deep_select_stride_requirement() - logits_row_bytes = num_sparse_cols * torch.bfloat16.itemsize - if num_sparse_cols >= 2**23 or logits_row_bytes % in_align != 0: - raise ValueError( - f"DeepSelect needs the sparse logits row ({num_sparse_cols} bf16 " - f"columns, {logits_row_bytes} B) to be {in_align}B-aligned." - ) - if topk_tokens > 4096 or (topk_tokens * torch.int32.itemsize) % out_align != 0: - raise ValueError( - f"DeepSelect needs topk <= 4096 with a {out_align}B-aligned int32 " - f"output row, got topk={topk_tokens}." - ) - - -_INT32_BIG = tl.constexpr(2**31 - 1) - - -@triton.jit(do_not_specialize_on_alignment=["end_ptr"]) -def _expand_candidates_kernel( - cand_ptr, - cand_stride, - ks_ptr, - ks_stride, - ke_ptr, - ke_stride, - si_ptr, - si_stride, - end_ptr, - end_stride, - K: tl.constexpr, - S: tl.constexpr, - RATIO: tl.constexpr, - SBK: tl.constexpr, -): - """One program per row: expand + filter + sort candidate blocks. - - Each row's valid sparse blocks come out sorted ascending (no dedup pass: - production candidates are unique, and the kernel tolerates repeats), - padded by repeating the last valid block — DeepGEMM's padding convention. - ``end`` counts valid sparse-token columns; valid columns form a prefix - because block ids are sorted. - """ - row = tl.program_id(0) - ks = tl.load(ks_ptr + row * ks_stride) - ke = tl.load(ke_ptr + row * ke_stride) - cols = tl.arange(0, K) - cand = tl.load(cand_ptr + row * cand_stride + cols) # K is pow2, no mask - if RATIO == 2: - cand = tl.interleave(cand * RATIO, cand * RATIO + 1) - - # Keep only blocks inside the row's KV range [0, ke - ks); the lower - # bound is free since candidates are non-negative. - kv_len = tl.maximum(ke - ks, 0) - max_block = (kv_len + SBK - 1) // SBK - valid = (cand >= 0) & (cand < max_block) - vals = tl.where(valid, cand, _INT32_BIG) - vals = tl.sort(vals) - - # Anchor at the row's ks rounded down to a sparse-block boundary - # (identity for the paged path, where ks == 0). - base = ks // SBK - si = tl.where(vals != _INT32_BIG, vals, 0) + base - n_valid = tl.sum(valid.to(tl.int32)) - last_rel = tl.max(tl.where(valid, cand, -1)) - pad_val = tl.where(n_valid > 0, last_rel + base, base) - tl.store( - si_ptr + row * si_stride + tl.arange(0, S), - tl.where(tl.arange(0, S) < n_valid, si, pad_val), - ) - - # Valid sparse-token column count; only the last block can be partial. - # Absolute start of the last block: (base + last_rel) * SBK + ks % SBK, - # which is exactly last_rel * SBK + ks. - last_start = last_rel * SBK + ks - fill = tl.minimum(tl.maximum(ke - last_start, 1), SBK) - end = tl.where(n_valid > 0, (n_valid - 1) * SBK + fill, 0) - tl.store(end_ptr + row * end_stride, end) - - -@triton.jit -def _sparse_topk_remap_kernel( - col_ptr, - col_stride, - si_ptr, - si_stride, - ks_ptr, - ks_stride, - out_ptr, - out_stride, - k, - width, - SBK: tl.constexpr, - K_POW2: tl.constexpr, -): - """Remap sparse top-k columns to request-local positions. - - Column ``j * SBK + o`` scores the token at - ``(si[row, j] - ks // SBK) * SBK + o``; columns outside ``[0, width)`` - (including DeepSelect's sentinel for NaN rows) become -1. - """ - row = tl.program_id(0) - ks_base = tl.load(ks_ptr + row * ks_stride) // SBK - cols = tl.arange(0, K_POW2) - c = tl.load(col_ptr + row * col_stride + cols, mask=cols < k, other=-1) - valid = (c >= 0) & (c < width) - cc = tl.where(valid, c, 0) - blocks = tl.load(si_ptr + row * si_stride + cc // SBK) - pos = (blocks - ks_base) * SBK + cc % SBK - tl.store(out_ptr + row * out_stride + cols, tl.where(valid, pos, -1), mask=cols < k) - - -def pick_sparse_block_kv(candidate_block_size: int) -> int: - """Largest sparse-block size (tokens) dividing the candidate block size.""" - for sbk in SPARSE_BLOCK_KV_CHOICES: - if candidate_block_size % sbk == 0: - return sbk - raise ValueError( - f"candidate_block_size={candidate_block_size} is not a multiple of " - f"any supported sparse_block_kv {SPARSE_BLOCK_KV_CHOICES}." - ) - - -def candidate_blocks_to_sparse_indices( - candidate_blocks: torch.Tensor, - row_ks: torch.Tensor, - row_ke: torch.Tensor, - candidate_block_size: int, - sparse_block_kv: int, - out: tuple[torch.Tensor, torch.Tensor] | None = None, -) -> tuple[torch.Tensor, torch.Tensor]: - """Expand request-local candidate blocks into DeepGEMM sparse block ids. - - Args: - candidate_blocks: [rows, K] int32 request-local candidate block ids - (-1 padded), in units of ``candidate_block_size`` positions. K - must be a power of two (the in-kernel sort); rows may be strided. - row_ks: [rows] int32 per-row K range start; bounds are in the same - (packed-workspace) coordinates the sparse kernel iterates over. - Pass zeros for the paged path, whose blocks are context-relative. - row_ke: [rows] int32 per-row K range end, in the same coordinates as - ``row_ks``. - candidate_block_size: Positions per candidate block. - sparse_block_kv: Positions per sparse block (8 or 16). - out: Optional ``(sparse_indices, end)`` buffers to write into, - shaped [rows, K * ratio] and [rows], both int32. - - Returns: - sparse_indices: [rows, K * ratio] int32 block ids for DeepGEMM, - sorted ascending per row with the valid prefix first (unique in - production, where candidates are unique) and the remaining slots - repeating the last valid block (the kernel's padding convention). - With the unaligned-ks variant, block ``i`` starts at - ``i * sparse_block_kv + ks % sparse_block_kv``. - end: [rows] int32 count of valid sparse-token columns; valid columns - always form a prefix because the block ids are sorted. - - """ - rows, num_candidates = candidate_blocks.shape - ratio = candidate_block_size // sparse_block_kv - assert ratio * sparse_block_kv == candidate_block_size and ratio in (1, 2), ( - candidate_block_size, - sparse_block_kv, - ) - assert num_candidates & (num_candidates - 1) == 0, num_candidates - num_sparse = num_candidates * ratio - if out is None: - device = candidate_blocks.device - out = ( - torch.empty(rows, num_sparse, dtype=torch.int32, device=device), - torch.empty(rows, dtype=torch.int32, device=device), - ) - sparse_indices, end = out - assert sparse_indices.shape == (rows, num_sparse) and end.shape == (rows,) - assert candidate_blocks.stride(1) == 1 and sparse_indices.stride(1) == 1 - for t in (candidate_blocks, row_ks, row_ke, sparse_indices, end): - assert t.dtype == torch.int32, t.dtype - - _expand_candidates_kernel[(rows,)]( - candidate_blocks, - candidate_blocks.stride(0), - row_ks, - row_ks.stride(0), - row_ke, - row_ke.stride(0), - sparse_indices, - sparse_indices.stride(0), - end, - end.stride(0), - K=num_candidates, - S=num_sparse, - RATIO=ratio, - SBK=sparse_block_kv, - num_warps=16, - ) - return sparse_indices, end - - -def sparse_topk_remap( - logits: torch.Tensor, - sparse_indices: torch.Tensor, - end: torch.Tensor, - row_ks: torch.Tensor, - sparse_block_kv: int, - topk_tokens: int, - topk_indices: torch.Tensor, - *, - col_indices: torch.Tensor, -) -> None: - """Row top-k over sparse logits, remapped to request-local positions. - - Column ``j * sparse_block_kv + o`` of ``logits`` scores the token at - request-local position - ``(sparse_indices[row, j] - ks // sparse_block_kv) * sparse_block_kv + o``. - - DeepSelect consumes the bf16 logits as they come out of the sparse - kernels; the valid columns form a prefix (block ids are sorted), so the - per-row ``end`` bounds drive it directly. Slots beyond a row's valid - count come back as -1. - - Args: - logits: [rows, width] bf16 sparse logits, as produced by the sparse - kernels. - sparse_indices: [rows, S] int32 sparse block ids backing ``logits``. - end: [rows] int32 per-row count of valid logit columns. - row_ks: [rows] int32 per-row K range start. - sparse_block_kv: Positions per sparse block (8 or 16). - topk_tokens: Number of tokens to keep per row. - topk_indices: [rows, topk_tokens] int32 output buffer. - col_indices: [rows, topk_tokens] int32 scratch for the sparse-column - top-k result (row stride 32B-aligned for DeepSelect). - - """ - rows, width = logits.shape - assert logits.dtype == torch.bfloat16 and width >= topk_tokens - assert end.dtype == torch.int32 and end.shape == (rows,) - cols = col_indices[:rows] - deep_select_topk(logits, topk_tokens, end=end, output_idx=cols) - _sparse_topk_remap_kernel[(rows,)]( - cols, - cols.stride(0), - sparse_indices, - sparse_indices.stride(0), - row_ks, - row_ks.stride(0), - topk_indices, - topk_indices.stride(0), - topk_tokens, - width, - SBK=sparse_block_kv, - K_POW2=triton.next_power_of_2(topk_tokens), - ) - - -def sparse_mqa_logits_prefill_chunk( - q: torch.Tensor, - q_scale: torch.Tensor, - k_quant: torch.Tensor, - k_scale: torch.Tensor, - weights: torch.Tensor, - cu_seqlen_ks: torch.Tensor, - cu_seqlen_ke: torch.Tensor, - candidate_blocks: torch.Tensor, - candidate_block_size: int, - sparse_block_kv: int, - topk_tokens: int, - topk_indices: torch.Tensor, - *, - sparse_indices: torch.Tensor, - end: torch.Tensor, - col_indices: torch.Tensor, - kernel_metadata: torch.Tensor | None = None, -) -> torch.Tensor: - """Sparse-MQA logits + top-k for one prefill chunk (packed KV workspace). - - Args: - q: [rows, H, D] packed Q values (MXFP4 viewed as int8). - q_scale: [rows, H] int32 packed UE8M0 Q scales. - k_quant: [total_kv, D] packed K workspace (MXFP4 viewed as int8). - k_scale: [total_kv] int32 packed UE8M0 K scales. - weights: [rows, H] bf16 per-head weights; the sparse kernels take - bf16 and do not fold the Q scale in. - cu_seqlen_ks: [rows] int32 per-token K start bounds in the packed - workspace. - cu_seqlen_ke: [rows] int32 per-token K end bounds in the packed - workspace. - candidate_blocks: [rows, K] int32 request-local candidate block ids. - candidate_block_size: Positions per candidate block. - sparse_block_kv: Positions per sparse block (8 or 16). - topk_tokens: Number of tokens to keep per row. - topk_indices: [rows, topk_tokens] output buffer. - sparse_indices: Caller-owned scratch, see - `candidate_blocks_to_sparse_indices`. - end: Caller-owned scratch, see `candidate_blocks_to_sparse_indices`. - col_indices: Caller-owned scratch, see `sparse_topk_remap`. - kernel_metadata: DeepGEMM schedule from a previous call with the same - candidates and bounds (i.e. another indexer layer in the same - step). When given, the candidate expansion is skipped and - ``sparse_indices``/``end`` are assumed to be up to date. - - Returns: - The DeepGEMM schedule metadata used, for reuse by later layers. - - """ - assert weights.dtype == torch.bfloat16, weights.dtype - if kernel_metadata is None: - candidate_blocks_to_sparse_indices( - candidate_blocks, - cu_seqlen_ks, - cu_seqlen_ke, - candidate_block_size, - sparse_block_kv, - out=(sparse_indices, end), - ) - kernel_metadata = get_sparse_mqa_logits_metadata( - cu_seqlen_ks, - cu_seqlen_ke, - k_quant.shape[0], - sparse_indices, - q.dtype, - sparse_block_kv, - ) - logits = fp8_fp4_sparse_mqa_logits( - (q, q_scale), - (k_quant, k_scale), - weights, - kernel_metadata, - sparse_indices.shape[1], - sparse_block_kv, - ) - sparse_topk_remap( - logits, - sparse_indices, - end, - cu_seqlen_ks, - sparse_block_kv, - topk_tokens, - topk_indices, - col_indices=col_indices, - ) - return kernel_metadata - - -def sparse_mqa_logits_paged_decode( - q: torch.Tensor, - q_scale: torch.Tensor, - kv_cache: torch.Tensor, - weights: torch.Tensor, - context_lens: torch.Tensor, - block_table: torch.Tensor, - row_indices: torch.Tensor, - candidate_blocks: torch.Tensor, - candidate_block_size: int, - sparse_block_kv: int, - topk_tokens: int, - topk_indices: torch.Tensor, - *, - row_ks: torch.Tensor, - sparse_indices: torch.Tensor, - end: torch.Tensor, - col_indices: torch.Tensor, - kernel_metadata: torch.Tensor | None = None, -) -> torch.Tensor: - """Sparse-MQA logits + top-k for paged decode rows. - - Every query is one row (spec-decode rows are flattened by the metadata - builder), so ``block_table`` is already per row. - - Args: - q: [rows, 1, H, D] packed Q values (MXFP4 viewed as int8). - q_scale: [rows, 1, H] int32 packed UE8M0 Q scales. - kv_cache: 4D [num_pages, page_kv, 1, head_bytes] uint8 fused cache - view; the page stride must be 256B-aligned. - weights: [rows, H] bf16 per-head weights. - context_lens: [rows] int32 per-row (compressed) context lengths. - block_table: [rows, P] int32 per-row page table, ``stride(-1) == 1``. - row_indices: [rows] int32 row -> request map (pairing only affects - scheduling). - candidate_blocks: [rows, K] int32 candidate block ids. - candidate_block_size: Positions per candidate block. - sparse_block_kv: Positions per sparse block (8 or 16). - topk_tokens: Number of tokens to keep per row. - topk_indices: [rows, topk_tokens] output buffer. - row_ks: [rows] int32 zeros (paged blocks are context-relative). - sparse_indices: Caller-owned scratch. - end: Caller-owned scratch. - col_indices: Caller-owned scratch. - kernel_metadata: See `sparse_mqa_logits_prefill_chunk`. - - Returns: - The DeepGEMM schedule metadata used, for reuse by later layers. - - """ - rows = q.shape[0] - page_kv = kv_cache.shape[1] - assert weights.dtype == torch.bfloat16, weights.dtype - assert kv_cache.stride(0) % 256 == 0, ( - "paged cache page stride must be 256B-aligned: " - f"shape={tuple(kv_cache.shape)} stride={kv_cache.stride()}" - ) - assert page_kv % sparse_block_kv == 0, (page_kv, sparse_block_kv) - assert block_table.shape[0] == rows and block_table.stride(-1) == 1 - assert context_lens.shape == (rows,) and context_lens.is_contiguous() - - if kernel_metadata is None: - candidate_blocks_to_sparse_indices( - candidate_blocks, - row_ks, - context_lens, - candidate_block_size, - sparse_block_kv, - out=(sparse_indices, end), - ) - kernel_metadata = get_paged_sparse_mqa_logits_metadata( - context_lens, - block_table, - row_indices, - page_kv, - sparse_indices, - q.dtype, - sparse_block_kv, - ) - logits = fp8_fp4_paged_sparse_mqa_logits( - (q, q_scale), - kv_cache, - weights, - kernel_metadata, - sparse_indices.shape[1], - sparse_block_kv, - ) - sparse_topk_remap( - logits, - sparse_indices, - end, - row_ks, - sparse_block_kv, - topk_tokens, - topk_indices, - col_indices=col_indices, - ) - return kernel_metadata diff --git a/vllm/model_executor/kernels/linear/cute_dsl/gemm_rs_ar.py b/vllm/model_executor/kernels/linear/cute_dsl/gemm_rs_ar.py deleted file mode 100644 index ec40fb056a..0000000000 --- a/vllm/model_executor/kernels/linear/cute_dsl/gemm_rs_ar.py +++ /dev/null @@ -1,1177 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -"""SM100 BF16/MXFP8 GEMM with fused tensor-parallel reduce-scatter/all-reduce. - -Model-neutral: any row-parallel projection whose output is reduced across TP -(Kimi-K3 ``o_proj``/``down_proj``, DeepSeek-V4.1 ``wo_b``) can bind to the -process-wide workspace through ``maybe_init_gemm_rs_ar`` + ``GemmRsAr.apply``. -""" - -# Based on CUTLASS's Blackwell distributed GEMM-RS example at dcf215a. -# See https://github.com/NVIDIA/cutlass/issues/3117 for memory semantics. -# The MXFP8 path follows the block-scaled SM100 GEMM template from -# https://github.com/gau-nernst/gn-kernels (sm100_mm_mxfp8.py). - -from functools import cache -from typing import TYPE_CHECKING - -import cutlass -import torch -import torch.distributed._symmetric_memory as symm_mem -from cuda.bindings.driver import CUstream -from cutlass import ( - BFloat16, - Float8E4M3FN, - Float8E8M0FNU, - Int32, - Int64, - Uint16, - Uint64, - cute, - utils, -) -from cutlass._mlir import ir -from cutlass._mlir.dialects import llvm, nvvm, vector -from cutlass.cute.nvgpu import cpasync, tcgen05 -from cutlass.cute.runtime import make_fake_stream, make_fake_tensor, make_ptr, nullptr -from cutlass.cutlass_dsl import dsl_user_op -from cutlass.utils import get_smem_capacity_in_bytes - -from vllm.cute_utils import _tcgen05, mbarrier, simple_tma_copy, to_cta0_smem -from vllm.distributed import get_tp_group -from vllm.logger import init_logger -from vllm.model_executor.layers.linear import LinearBase, UnquantizedLinearMethod -from vllm.platforms import current_platform - -if TYPE_CHECKING: - from vllm.config import VllmConfig - -logger = init_logger(__name__) - - -@dsl_user_op -def nanosleep(ns: int, *, loc=None, ip=None) -> None: - nvvm.nanosleep(Int32(ns).ir_value(loc=loc, ip=ip), loc=loc, ip=ip) - - -@dsl_user_op -def multimem_ld_reduce_16B(x: cute.Tensor, *, loc=None, ip=None) -> cute.Tensor: - # The vector instruction assumes x is contiguous and 16-byte aligned. - assert x.element_type == BFloat16 - vec_type = ".v4.bf16x2" - - ptr = x.iterator.toint(loc=loc, ip=ip).ir_value(loc=loc, ip=ip) - asm = ( - "multimem.ld_reduce.relaxed.gpu.global.add.acc::f32" - f"{vec_type} {{$0, $1, $2, $3}}, [$4];" - ) - struct = llvm.inline_asm( - llvm.StructType.get_literal([Int32.mlir_type] * 4), - [ptr], - asm, - "=r,=r,=r,=r,l", - has_side_effects=True, - loc=loc, - ip=ip, - ) - vec = vector.from_elements( - ir.VectorType.get([4], Int32.mlir_type, loc=loc), - [ - llvm.extractvalue(Int32.mlir_type, struct, [i], loc=loc, ip=ip) - for i in range(4) - ], - loc=loc, - ip=ip, - ) - ssa = cute.TensorSSA(vec, 4, Int32) - - y = cute.make_rmem_tensor(4, Int32) - y.store(ssa) - return cute.recast_tensor(y, x.element_type) - - -@dsl_user_op -def multimem_st_16B(dst: cute.Tensor, value: cute.Tensor, *, loc=None, ip=None) -> None: - # The vector instruction assumes dst and value are contiguous BF16x8 vectors - # and dst is 16-byte aligned. - assert dst.element_type == BFloat16 - assert value.element_type == BFloat16 - ptr = dst.iterator.toint(loc=loc, ip=ip).ir_value(loc=loc, ip=ip) - regs = cute.recast_tensor(value, Int32, loc=loc, ip=ip) - llvm.inline_asm( - None, - [ptr] + [regs[i].ir_value(loc=loc, ip=ip) for i in range(4)], - "multimem.st.relaxed.sys.global.v4.f32 [$0], {$1, $2, $3, $4};", - "l,r,r,r,r", - has_side_effects=True, - loc=loc, - ip=ip, - ) - - -class Sm100GemmRsAr: - def __init__( - self, - rank: int, - num_ranks: int, - cta_group: int = 1, - all_reduce: bool = False, - dtype: str = "bf16", - ) -> None: - assert dtype in ("bf16", "mxfp8") - self.rank = rank - self.num_ranks = num_ranks - BM = 128 - BN = 128 - BK = {"bf16": 64, "mxfp8": 128}[dtype] # 128B - self.cta_tile = (BM, BN, BK) - self.cta_group = cta_group - self.all_reduce = all_reduce - self.dtype = dtype - - smem_bytes = get_smem_capacity_in_bytes() - if dtype == "bf16": - self.stage_size = (BM + (BN // cta_group)) * BK * 2 - elif dtype == "mxfp8": - self.stage_size = (BM + (BN // cta_group)) * BK + (BM + BN) * (BK // 32) - self.num_stages = smem_bytes // self.stage_size - - @cute.jit - def prepare_tma( - self, tensor: cute.Tensor, BM: cutlass.Constexpr, BK: cutlass.Constexpr - ) -> cpasync.TmaInfo: - tma_group = ( - tcgen05.CtaGroup.TWO if self.cta_group == 2 else tcgen05.CtaGroup.ONE - ) - tma_op = cpasync.CopyBulkTensorTileG2SOp(cta_group=tma_group) - swizzle_128b = cute.make_swizzle(3, 4, 3) - layout = cute.make_layout( - (BM, BK, self.num_stages), - stride=(BK, 1, BM * BK), - ) - layout = cute.make_composed_layout(swizzle_128b, 0, layout) - return cpasync.make_tiled_tma_atom(tma_op, tensor, layout, (BM, BK)) - - @cute.jit - def prepare_sf_tma( - self, - SF: cute.Tensor, - R: Int32, - Ksf: Int32, - BR: cutlass.Constexpr, - BKsf: cutlass.Constexpr, - ) -> cpasync.TmaInfo: - tma_group = ( - tcgen05.CtaGroup.TWO if self.cta_group == 2 else tcgen05.CtaGroup.ONE - ) - tma_op = cpasync.CopyBulkTensorTileG2SOp(cta_group=tma_group) - # NVIDIA SF gmem layout: an [R, Ksf] scale grid padded to R % 128 == 0 - # and permuted as [R/128, Ksf/4, 32, 4, 4]. The trailing [32, 4, 4] - # atom is the opaque 512B block tcgen05 expects, so treat it as bytes. - # The Int64 recast keeps the TMA boxDim <= 256. - g_layout = cute.make_layout((Ksf * 128, cute.ceil_div(R, 128))) - SF_i64 = cute.recast_tensor(cute.make_tensor(SF.iterator, g_layout), Int64) - tiler = (128 * BKsf // 8, BR // 128) - layout = cute.make_layout((*tiler, self.num_stages)) - return cpasync.make_tiled_tma_atom(tma_op, SF_i64, layout, tiler) - - @cute.jit - def __call__( - self, - A: cute.Tensor, - B: cute.Tensor, - SFA: cute.Tensor, - SFB: cute.Tensor, - partial_uc: cute.Tensor, - partial_mc_ptr: cute.Pointer, - output: cute.Tensor, - flags_uc: cute.Tensor, - flags_mc_ptr: cute.Pointer, - peer_flag_ptr: cute.Pointer, - grid_size: Int32, - stream: CUstream, - ) -> None: - M, K = A.shape - N = B.shape[0] - BM, BN, BK = self.cta_tile - A_tma = self.prepare_tma(A, BM, BK) - B_tma = self.prepare_tma(B, BN // self.cta_group, BK) - if cutlass.const_expr(self.dtype == "mxfp8"): - SFA_tma = self.prepare_sf_tma(SFA, M, K // 32, BM, BK // 32) - SFB_tma = self.prepare_sf_tma(SFB, N, K // 32, BN, BK // 32) - else: - SFA_tma = None - SFB_tma = None - padded_M = partial_uc.shape[0] - partial_mc = cute.make_tensor( - partial_mc_ptr, - cute.make_layout((padded_M, N), stride=(N, 1)), - ) - peer_flags = cute.make_tensor( - peer_flag_ptr, - cute.make_layout(self.num_ranks), - ) - - grid = (grid_size, 1, 1) - block = (10 * 32, 1, 1) - cluster = (self.cta_group, 1, 1) - self.kernel( - A_tma, - B_tma, - SFA_tma, - SFB_tma, - partial_uc, - partial_mc, - output, - flags_uc.iterator, - flags_mc_ptr, - peer_flags, - ).launch(grid=grid, block=block, cluster=cluster, stream=stream) - - @cute.kernel - def kernel( - self, - A_tma: cpasync.TmaInfo, - B_tma: cpasync.TmaInfo, - SFA_tma: cpasync.TmaInfo, - SFB_tma: cpasync.TmaInfo, - partial_uc: cute.Tensor, - partial_mc: cute.Tensor, - output: cute.Tensor, - flags_uc_ptr: cute.Pointer, - flags_mc_ptr: cute.Pointer, - peer_flags: cute.Tensor, - ) -> None: - tid, _, _ = cute.arch.thread_idx() - raw_bid, _, _ = cute.arch.block_idx() - num_bids, _, _ = cute.arch.grid_dim() - warp_id = cute.arch.make_warp_uniform(tid // 32) - - BM, BN, BK = self.cta_tile - cta_group = self.cta_group - num_stages = self.num_stages - num_ranks = self.num_ranks - - is_2cta = cta_group == 2 - cta_rank = raw_bid % self.cta_group - - # MXFP8 reserves the last 16 tmem columns (496-512) for the A/B scale - # factors. - num_tmem_stages = (496 if self.dtype == "mxfp8" else 512) // BN - - smem = utils.SmemAllocator() - sA = smem.allocate_tensor( - A_tma.atom.value_type, - A_tma.smem_layout.outer, - byte_alignment=128, - swizzle=A_tma.smem_layout.inner, - ) - sB = smem.allocate_tensor( - B_tma.atom.value_type, - B_tma.smem_layout.outer, - byte_alignment=128, - swizzle=B_tma.smem_layout.inner, - ) - if cutlass.const_expr(self.dtype == "mxfp8"): - sSFA = smem.allocate_tensor(Int64, SFA_tma.smem_layout, byte_alignment=128) - sSFB = smem.allocate_tensor(Int64, SFB_tma.smem_layout, byte_alignment=128) - - tma_full_mbar = smem.allocate_array(Int64, num_stages) - tma_empty_mbar = smem.allocate_array(Int64, num_stages) - tmem_full_mbar = smem.allocate_array(Int64, num_tmem_stages) - tmem_empty_mbar = smem.allocate_array(Int64, num_tmem_stages) - - taddr = smem.allocate(Int32, 4) - - # Named barriers - BAR_TMEM_ALLOC = 1 - BAR_EPILOGUE = 2 - BAR_COMM = 3 - - M, K = A_tma.tma_tensor.shape - N, _ = B_tma.tma_tensor.shape - local_M = ( - partial_uc.shape[0] // num_ranks if self.all_reduce else output.shape[0] - ) - grid_m = cute.ceil_div(M, BM) - # Keep 2-CTA clusters within a single N tile. - grid_m = cute.ceil_div(grid_m, cta_group) * cta_group - grid_n = cute.ceil_div(N, BN) - - if warp_id == 0: - with cute.arch.elect_one(): - for i in cutlass.range_constexpr(num_stages): - cute.arch.mbarrier_init(tma_full_mbar + i, cta_group) - cute.arch.mbarrier_init(tma_empty_mbar + i, 1) - for i in cutlass.range_constexpr(num_tmem_stages): - cute.arch.mbarrier_init(tmem_full_mbar + i, 1) - cute.arch.mbarrier_init(tmem_empty_mbar + i, 128 * cta_group) - cute.arch.mbarrier_init_fence() - elif warp_id == 1: - cpasync.prefetch_descriptor(A_tma.atom) - cpasync.prefetch_descriptor(B_tma.atom) - if cutlass.const_expr(self.dtype == "mxfp8"): - cpasync.prefetch_descriptor(SFA_tma.atom) - cpasync.prefetch_descriptor(SFB_tma.atom) - - if cutlass.const_expr(is_2cta): - cute.arch.cluster_arrive_relaxed() - cute.arch.cluster_wait() - else: - cute.arch.sync_threads() - - total_tiles = grid_m * grid_n - - if warp_id == 9: - # TMA warp - tma_stage = 0 - parity = 1 - - if cutlass.const_expr(is_2cta): - tma_full_mbar_ = to_cta0_smem(tma_full_mbar) - else: - tma_full_mbar_ = tma_full_mbar - - # Select global-memory tiles. - # [(BM, BK), (M/BM, K/BK)] - gA_tiles = cute.zipped_divide(A_tma.tma_tensor, (BM, BK)) - gB_tiles = cute.zipped_divide(B_tma.tma_tensor, (BN // cta_group, BK)) - if cutlass.const_expr(self.dtype == "mxfp8"): - BKsf = BK // 32 - gSFA_tiles = cute.zipped_divide( - SFA_tma.tma_tensor, (128 * BKsf // 8, BM // 128) - ) - gSFB_tiles = cute.zipped_divide( - SFB_tma.tma_tensor, (128 * BKsf // 8, BN // 128) - ) - - for bid in range(raw_bid, total_tiles, num_bids): - bid_m = bid % grid_m - bid_n = bid // grid_m - if cutlass.const_expr(cta_group == 2): - bid_n_b = bid_n * cta_group + cta_rank - else: - bid_n_b = bid_n - - for iter_k in cutlass.range(cute.ceil_div(K, BK), unroll=1): - mbar = tma_full_mbar_ + tma_stage - cute.arch.mbarrier_wait(tma_empty_mbar + tma_stage, parity) - - with cute.arch.elect_one(): - mbarrier.arrive_expect_tx(mbar, self.stage_size, "cluster") - simple_tma_copy( - A_tma.atom, - gA_tiles[None, (bid_m, iter_k)], - sA[None, None, tma_stage], - mbar, - ) - simple_tma_copy( - B_tma.atom, - gB_tiles[None, (bid_n_b, iter_k)], - sB[None, None, tma_stage], - mbar, - ) - if cutlass.const_expr(self.dtype == "mxfp8"): - simple_tma_copy( - SFA_tma.atom, - gSFA_tiles[None, (iter_k, bid_m)], - sSFA[None, None, tma_stage], - mbar, - ) - simple_tma_copy( - SFB_tma.atom, - gSFB_tiles[None, (iter_k, bid_n)], - sSFB[None, None, tma_stage], - mbar, - ) - - tma_stage = (tma_stage + 1) % num_stages - if tma_stage == 0: - parity ^= 1 - - elif warp_id == 8: - # MMA warp - cute.arch.barrier(barrier_id=BAR_TMEM_ALLOC, number_of_threads=5 * 32) - - if cta_rank == 0: - tma_stage = 0 - tma_full_parity = 0 - tmem_stage = 0 - tmem_empty_parity = 1 - - MMA_M = BM * cta_group - MMA_N = BN - sdesc = _tcgen05.make_sdesc_128B_swizzle(0) - if cutlass.const_expr(self.dtype == "mxfp8"): - # SF smem descriptor: no swizzle, SBO = 8 * 16 bytes. - sdesc_sf = Uint64((8 << 32) | (1 << 46)) - sfa_tmem = 496 - sfb_tmem = sfa_tmem + 4 - idesc = _tcgen05.make_mxfp8_idesc(MMA_M, MMA_N) - else: - idesc = _tcgen05.make_bf16_idesc(MMA_M, MMA_N) - multicast_mask = Uint16((1 << self.cta_group) - 1) - - for bid in range(raw_bid, total_tiles, num_bids): - cute.arch.mbarrier_wait( - tmem_empty_mbar + tmem_stage, tmem_empty_parity - ) - _tcgen05.fence_after_thread_sync() - - for iter_k in cutlass.range(cute.ceil_div(K, BK), unroll=1): - d_tmem = BN * tmem_stage - a_addr = sA[None, None, tma_stage].iterator.toint() - b_addr = sB[None, None, tma_stage].iterator.toint() - a_desc = sdesc | (a_addr >> 4) - b_desc = sdesc | (b_addr >> 4) - if cutlass.const_expr(self.dtype == "mxfp8"): - sfa_desc = sdesc_sf | ( - sSFA[None, None, tma_stage].iterator.toint() >> 4 - ) - sfb_desc = sdesc_sf | ( - sSFB[None, None, tma_stage].iterator.toint() >> 4 - ) - - cute.arch.mbarrier_wait( - tma_full_mbar + tma_stage, tma_full_parity - ) - _tcgen05.fence_after_thread_sync() - - # load SF from smem->tmem - if cutlass.const_expr(self.dtype == "mxfp8"): - _tcgen05.cp( - sfa_tmem, sfa_desc, "32x128b", "warpx4", cta_group - ) - for j in cutlass.range_constexpr(BN // 128): - _tcgen05.cp( - sfb_tmem + j * 4, - sfb_desc, - "32x128b", - "warpx4", - cta_group, - ) - sfb_desc += (128 * 4) >> 4 - - # 4 = 128B / 32B - for mma_k in cutlass.range_constexpr(4): - enable_d = iter_k > 0 or mma_k > 0 - if cutlass.const_expr(self.dtype == "mxfp8"): - # select SF IDs - idesc_k = idesc + ((mma_k << 4) | (mma_k << 29)) - _tcgen05.mma_mxfp8( - d_tmem, - a_desc, - b_desc, - idesc_k, - sfa_tmem, - sfb_tmem, - enable_d, - cta_group, - ) - else: - _tcgen05.mma_f16( - d_tmem, a_desc, b_desc, idesc, enable_d, cta_group - ) - a_desc += 32 >> 4 - b_desc += 32 >> 4 - _tcgen05.commit( - tma_empty_mbar + tma_stage, multicast_mask, cta_group - ) - - tma_stage = (tma_stage + 1) % num_stages - if tma_stage == 0: - tma_full_parity ^= 1 - - _tcgen05.commit( - tmem_full_mbar + tmem_stage, multicast_mask, cta_group - ) - - tmem_stage = (tmem_stage + 1) % num_tmem_stages - if tmem_stage == 0: - tmem_empty_parity ^= 1 - - elif warp_id >= 4: - # Communication warps - # Keep epilogue in warps 0-3 and communication in warps 4-7. - # Swapping the warpgroups can hang due to warp scheduling. - tid_ = tid % 128 - - # Offset in the [M, N] GEMM result. - rank_start = self.rank * local_M - rank_end = min(rank_start + local_M, M) - - st_atom = cute.make_copy_atom( - cute.nvgpu.CopyUniversalOp(), BFloat16, num_bits_per_copy=128 - ) - - # Each thread issues one BF16x8 multimem reduction per vector. - vec_width = 8 - vec_cols = BN // vec_width - max_tile_rows = BM // num_ranks - vecs_per_tile = max_tile_rows * vec_cols - partial_vecs = cute.zipped_divide(partial_mc, (1, vec_width)) - if cutlass.const_expr(not self.all_reduce): - output_vecs = cute.zipped_divide(output, (1, vec_width)) - - for tile_id in range(raw_bid, total_tiles, num_bids): - rs_bid_m = tile_id % grid_m - bid_n = tile_id // grid_m - - local_row_start = rs_bid_m * local_M // grid_m - local_row_end = (rs_bid_m + 1) * local_M // grid_m - global_row_start = rank_start + local_row_start - global_row_end = min(rank_start + local_row_end, M) - - if global_row_start < M: - # Map the current RS tile to the required GEMM tiles. - # Since an RS tile is smaller than a GEMM tile, each RS tile - # can overlap at most 2 GEMM tiles. - gemm_bid_m0 = global_row_start // BM - gemm_bid_m1 = max(global_row_end - 1, global_row_start) // BM - - # Poll local L2 with relaxed GPU-scope loads. The following - # multimem reduction reads every rank's L2 directly, so no - # acquire fence or L1 invalidation is needed. - if tid_ == 0: - # Poll the 1st GEMM tile. - flag_ptr = flags_uc_ptr + bid_n * grid_m + gemm_bid_m0 - arrivals = cute.arch.load( - flag_ptr, Int32, sem="relaxed", scope="gpu" - ) - while arrivals < num_ranks: - nanosleep(64) - arrivals = cute.arch.load( - flag_ptr, Int32, sem="relaxed", scope="gpu" - ) - elif tid_ == 32 and gemm_bid_m1 != gemm_bid_m0: - # Poll the 2nd GEMM tile with another warp if needed. - flag_ptr = flags_uc_ptr + bid_n * grid_m + gemm_bid_m1 - arrivals = cute.arch.load( - flag_ptr, Int32, sem="relaxed", scope="gpu" - ) - while arrivals < num_ranks: - nanosleep(64) - arrivals = cute.arch.load( - flag_ptr, Int32, sem="relaxed", scope="gpu" - ) - cute.arch.barrier(barrier_id=BAR_COMM, number_of_threads=128) - - # Issue multimem.ld_reduce before storing any result. - reduced_vecs = [] - for vec_iter in cutlass.range_constexpr(vecs_per_tile // 128): - vec_idx = tid_ + vec_iter * 128 - - local_row = local_row_start + vec_idx // vec_cols - global_row = rank_start + local_row - col = bid_n * vec_cols + vec_idx % vec_cols - - reduced_vec = cute.make_rmem_tensor(vec_width, BFloat16) - if local_row < local_row_end and global_row < M: - tmp = multimem_ld_reduce_16B( - partial_vecs[None, (global_row, col)] - ) - reduced_vec.store(tmp.load()) - reduced_vecs.append(reduced_vec) - - for vec_iter in cutlass.range_constexpr(vecs_per_tile // 128): - vec_idx = tid_ + vec_iter * 128 - - local_row = local_row_start + vec_idx // vec_cols - global_row = rank_start + local_row - col = bid_n * vec_cols + vec_idx % vec_cols - - if local_row < local_row_end and global_row < M: - if cutlass.const_expr(self.all_reduce): - # Broadcast the result to all ranks. - multimem_st_16B( - partial_vecs[None, (global_row, col)], - reduced_vecs[vec_iter], - ) - else: - # Store the result to local L2. - cute.copy( - st_atom, - reduced_vecs[vec_iter], - output_vecs[None, (local_row, col)], - ) - cute.arch.barrier(barrier_id=BAR_COMM, number_of_threads=128) - - # Release each GEMM tile consumed by this RS tile. The last - # consumer resets its producer flag for the next launch. - def release_gemm_tile( - gemm_bid_m, - tile_M, - logical_M, - rank_start, - rank_end, - local_M, - grid_m, - tile_N, - flags, - num_ranks, - ): - gemm_start = gemm_bid_m * tile_M - gemm_end = min(gemm_start + tile_M, logical_M) - local_start = max(gemm_start, rank_start) - rank_start - local_end = min(gemm_end, rank_end) - rank_start - - # Compute the number of consumers to identify the last - # arrival, which resets the producer flag. - first_rs_tile = ( - cute.ceil_div((local_start + 1) * grid_m, local_M) - 1 - ) - last_rs_tile = cute.ceil_div(local_end * grid_m, local_M) - 1 - num_consumers = last_rs_tile - first_rs_tile + 1 - - flag_ptr = flags + tile_N * grid_m + gemm_bid_m - old_count = cute.arch.atomic_add( - flag_ptr, Int32(1), sem="relaxed", scope="gpu" - ) - if old_count == num_ranks + num_consumers - 1: - cute.arch.store( - flag_ptr, Int32(0), sem="relaxed", scope="gpu" - ) - - if tid_ == 0: - # Arrive at the 1st GEMM tile. - release_gemm_tile( - gemm_bid_m0, - BM, - M, - rank_start, - rank_end, - local_M, - grid_m, - bid_n, - flags_uc_ptr, - num_ranks, - ) - elif tid_ == 32 and gemm_bid_m1 != gemm_bid_m0: - # Arrive at the 2nd GEMM tile if necessary. - release_gemm_tile( - gemm_bid_m1, - BM, - M, - rank_start, - rank_end, - local_M, - grid_m, - bid_n, - flags_uc_ptr, - num_ranks, - ) - - # Exit barrier. GPU scope is sufficient for RS because the kernel - # writes local global memory and only needs to flush it to local L2. - # AR uses system scope to flush the multimem stores to remote L2. - cute.arch.barrier(barrier_id=BAR_COMM, number_of_threads=128) - if tid_ == 0: - exit_flag = total_tiles + raw_bid - scope = "sys" if self.all_reduce else "gpu" - utils.distributed.multimem_red_add1( - flags_mc_ptr + exit_flag, order="release", scope=scope - ) - utils.distributed.spin_lock_atom_cas_acquire_wait( - flags_uc_ptr + exit_flag, - expected_val=num_ranks, - reset_val=0, - scope=scope, - ) - - else: - # Epilogue warps - warp_id_ = warp_id % 4 - tid_ = tid % 128 - - peer_flag_bases = cute.make_rmem_tensor(num_ranks, Int64) - if tid_ == 0: - for rank in cutlass.range_constexpr(num_ranks): - peer_flag_bases[rank] = cute.arch.load( - (peer_flags.iterator + rank).llvm_ptr, - Int64, - ) - - if warp_id_ == 0: - _tcgen05.alloc(taddr, cta_group) - cute.arch.barrier(barrier_id=BAR_TMEM_ALLOC, number_of_threads=5 * 32) - - WIDTH = cutlass.const_expr(16) - partial_vecs = cute.zipped_divide(partial_uc, (1, WIDTH)) - - bf16x16_atom = cute.make_copy_atom( - cute.nvgpu.CopyUniversalOp(), - BFloat16, - num_bits_per_copy=256, - l1c_evict_priority=cute.nvgpu.CacheEvictionPriority.NO_ALLOCATE, - ) - - tmem_stage = 0 - parity = 0 - - if cutlass.const_expr(is_2cta): - tmem_empty_mbar_ = to_cta0_smem(tmem_empty_mbar) - else: - tmem_empty_mbar_ = tmem_empty_mbar - - for bid in range(raw_bid, total_tiles, num_bids): - bid_m = bid % grid_m - bid_n = bid // grid_m - - if warp_id_ == 0: - cute.arch.mbarrier_wait(tmem_full_mbar + tmem_stage, parity) - cute.arch.barrier(barrier_id=BAR_EPILOGUE, number_of_threads=128) - _tcgen05.fence_after_thread_sync() - - for i in cutlass.range_constexpr(BN // WIDTH): - tcol = tmem_stage * BN + i * WIDTH - regs = _tcgen05.ld(warp_id_ * 32, tcol, "32x32b", WIDTH) - _tcgen05.wait_ld() - - if cutlass.const_expr(i == BN // WIDTH - 1): - _tcgen05.fence_before_thread_sync() - mbarrier.arrive(tmem_empty_mbar_ + tmem_stage, "cluster") - - tmp = cute.make_rmem_tensor(WIDTH, BFloat16) - tmp.store(regs.to(BFloat16)) - - global_row = bid_m * BM + tid_ - if global_row < M: - coord = (global_row, bid_n * (BN // WIDTH) + i) - cute.copy(bf16x16_atom, tmp, partial_vecs[None, coord]) - - cute.arch.barrier(barrier_id=BAR_EPILOGUE, number_of_threads=128) - - # Signal GEMM completion. GPU scope is sufficient because the - # partial output only needs to be flushed to local L2. - if tid_ == 0 and bid_m * BM < M: - gemm_start = bid_m * BM - gemm_end = min(gemm_start + BM, M) - first_owner = gemm_start // local_M - last_owner = (gemm_end - 1) // local_M - - # signal to all consuming ranks (can be more than 1) - for rank in cutlass.range_constexpr(num_ranks): - if first_owner <= rank and rank <= last_owner: - ptr = cute.make_ptr( - Int32, - peer_flag_bases[rank], - cute.AddressSpace.gmem, - assumed_align=16, - ) - ptr += bid_m + bid_n * grid_m - utils.distributed.red_add1( - ptr, order="release", scope="gpu" - ) - - tmem_stage = (tmem_stage + 1) % num_tmem_stages - if tmem_stage == 0: - parity ^= 1 - - if cutlass.const_expr(is_2cta): - cute.arch.cluster_arrive_relaxed() - cute.arch.cluster_wait() - else: - cute.arch.barrier(barrier_id=BAR_EPILOGUE, number_of_threads=128) - if warp_id_ == 0: - _tcgen05.dealloc(cta_group) - - @cache - @staticmethod - def compile( - rank: int, - num_ranks: int, - cta_group: int, - all_reduce: bool = False, - dtype: str = "bf16", - ): - M = cute.sym_int() - padded_M = cute.sym_int() - N = cute.sym_int() - K = cute.sym_int() - local_M = cute.sym_int() - num_flags = cute.sym_int() - - ab_dtype = Float8E4M3FN if dtype == "mxfp8" else BFloat16 - # Row stride must keep rows 16B-aligned: 8 BF16 or 16 FP8 elements. - row_div = 16 if dtype == "mxfp8" else 8 - A = make_fake_tensor( - ab_dtype, - (M, K), - (cute.sym_int64(divisibility=row_div), 1), - assumed_align=16, - ) - B = make_fake_tensor( - ab_dtype, - (N, K), - (cute.sym_int64(divisibility=row_div), 1), - assumed_align=16, - ) - if dtype == "mxfp8": - SFA = make_fake_tensor( - Float8E8M0FNU, - (cute.sym_int(divisibility=512),), - (1,), - assumed_align=16, - ) - SFB = make_fake_tensor( - Float8E8M0FNU, - (cute.sym_int(divisibility=512),), - (1,), - assumed_align=16, - ) - else: - SFA = None - SFB = None - partial = make_fake_tensor( - BFloat16, - (padded_M, N), - (cute.sym_int64(divisibility=16), 1), - assumed_align=32, - ) - partial_mc_ptr = nullptr(BFloat16, cute.AddressSpace.gmem, assumed_align=32) - output = ( - None - if all_reduce - else make_fake_tensor( - BFloat16, - (local_M, N), - (cute.sym_int64(divisibility=16), 1), - assumed_align=32, - ) - ) - flags = make_fake_tensor(Int32, (num_flags,), (1,), assumed_align=16) - flags_mc_ptr = nullptr(Int32, cute.AddressSpace.gmem, assumed_align=16) - peer_flag_ptr = nullptr(Int64, cute.AddressSpace.gmem, assumed_align=8) - - stream = make_fake_stream(use_tvm_ffi_env_stream=True) - kernel = Sm100GemmRsAr( - rank, num_ranks, cta_group, all_reduce=all_reduce, dtype=dtype - ) - return cute.compile( - kernel, - A, - B, - SFA, - SFB, - partial, - partial_mc_ptr, - output, - flags, - flags_mc_ptr, - peer_flag_ptr, - 128, - stream, - options="--enable-tvm-ffi", - ) - - -class GemmRsAr: - """Own the symmetric workspace for GEMM-RS/AR launches. - - All TP ranks must belong to one NVLink domain for multimem instructions. - - Each instance is bound to either RS or AR. A vLLM worker has one static - sequence-parallel topology, so the process-wide singleton only needs one - mode. Two independent mode-specific singletons would lift that restriction - but duplicate the large symmetric workspace. A future mixed-mode design - should instead use lightweight RS/AR frontends over one shared multicast - workspace; that is outside this integration's current scope. - """ - - def __init__(self, *, max_M: int, N: int, all_reduce: bool = False) -> None: - tp_group = get_tp_group() - group = tp_group.device_group - rank = tp_group.rank_in_group - world_size = tp_group.world_size - device = torch.device("cuda", torch.accelerator.current_device_index()) - - assert 1 < world_size <= 16 - assert 128 % world_size == 0 - assert max_M >= 128 and N % 128 == 0 - - max_M = (max_M + world_size - 1) // world_size * world_size - self.rank = rank - self.world_size = world_size - self.max_M = max_M - self.N = N - self.device = device - self.all_reduce = all_reduce - self.dtypes: set[str] = set() - - self.partial = symm_mem.empty((max_M, N), dtype=torch.bfloat16, device=device) - self.partial_handle = symm_mem.rendezvous(self.partial, group) - if self.partial_handle.multicast_ptr == 0: - raise RuntimeError("GEMM-RS/AR requires NVLink multicast memory") - self.partial_mc_ptr = make_ptr( - BFloat16, - self.partial_handle.multicast_ptr, - cute.AddressSpace.gmem, - assumed_align=32, - ) - - grid_m = (max_M + 127) // 128 - cta_group = 2 if max_M >= 1024 or grid_m % 2 == 0 else 1 - grid_m = (grid_m + cta_group - 1) // cta_group * cta_group - self.num_sms = torch.cuda.get_device_properties(device).multi_processor_count - max_flags = grid_m * (N // 128) + self.num_sms - self.flags = symm_mem.empty(max_flags, dtype=torch.int32, device=device) - self.flags_handle = symm_mem.rendezvous(self.flags, group) - if self.flags_handle.multicast_ptr == 0: - raise RuntimeError("GEMM-RS/AR requires NVLink multicast memory") - self.flags.zero_() - self.flags_mc_ptr = make_ptr( - Int32, - self.flags_handle.multicast_ptr, - cute.AddressSpace.gmem, - assumed_align=16, - ) - self.peer_flag_ptr = make_ptr( - Int64, - self.flags_handle.buffer_ptrs_dev, - cute.AddressSpace.gmem, - assumed_align=8, - ) - - torch.accelerator.synchronize(device) - tp_group.barrier() - - def can_run(self, linear: LinearBase) -> bool: - from vllm.model_executor.kernels.linear.mxfp8.flashinfer import ( - FlashInferCutedslMxfp8LinearKernel, - FlashInferCutlassMxfp8LinearKernel, - ) - - # Called before weight loading; online quantization starts on meta. - if isinstance(linear.quant_method, UnquantizedLinearMethod): - dtype, k_alignment = "bf16", 64 - if linear.weight.dtype != torch.bfloat16: - return False - else: - method = getattr(linear, "scheme", linear.quant_method) - kernel = getattr(method, "kernel", None) - if not isinstance( - kernel, - ( - FlashInferCutedslMxfp8LinearKernel, - FlashInferCutlassMxfp8LinearKernel, - ), - ): - return False - dtype, k_alignment = "mxfp8", 128 - - w = linear.weight - if ( - w.ndim != 2 - or w.shape[0] != self.N - or w.shape[1] % k_alignment != 0 - or (w.device != self.device and not w.is_meta) - or not w.is_contiguous() - ): - return False - self.dtypes.add(dtype) - return True - - def warn_incompatible_projection(self) -> None: - logger.warning_once( - "Some projections are incompatible with GEMM-RS/AR; using the " - "unfused path instead.", - scope="global", - ) - - def should_run(self, x: torch.Tensor) -> bool: - # Use the same threshold for RS and AR for now. Small-M shapes are - # supported but faster on the existing LL path. - return x.shape[0] >= 128 - - def apply(self, x: torch.Tensor, linear: LinearBase) -> torch.Tensor: - from vllm.model_executor.kernels.linear.mxfp8.flashinfer import ( - FlashInferCutedslMxfp8LinearKernel, - ) - - method = getattr(linear, "scheme", linear.quant_method) - w = linear.weight - if isinstance( - getattr(method, "kernel", None), FlashInferCutedslMxfp8LinearKernel - ): - # This backend stores a column-major [K, N] view after loading. - w = w.t() - w_sf = getattr(linear, "weight_scale", None) - - assert x.ndim == 2 - M, K = x.shape - assert 0 < M <= self.max_M - assert x.dtype == torch.bfloat16 - assert x.device == self.device - assert x.is_contiguous() - assert w.device == self.device - N = self.N - dtype = "mxfp8" if w.dtype == torch.float8_e4m3fn else "bf16" - x_sf = None - if dtype == "mxfp8": - from vllm.model_executor.layers.quantization.utils.mxfp8_utils import ( - mxfp8_e4m3_quantize, - ) - - assert w_sf is not None - assert w_sf.dtype == torch.uint8 and w_sf.device == self.device - assert w_sf.ndim == 1 and w_sf.is_contiguous() - assert w_sf.numel() == N * (K // 32) - assert w.shape == (N, K) and K % 128 == 0 - assert w.is_contiguous() - x, x_sf = mxfp8_e4m3_quantize(x, is_sf_swizzled_layout=True) - # The kernel declares the scales as E8M0; the stored/quantized - # tensors carry the same bits as uint8. - x_sf = x_sf.view(torch.float8_e8m0fnu) - w_sf = w_sf.view(torch.float8_e8m0fnu) - else: - assert w_sf is None - assert w.shape == (N, K) and K % 64 == 0 - assert w.dtype == torch.bfloat16 - assert w.is_contiguous() - padded_M = (M + self.world_size - 1) // self.world_size - padded_M *= self.world_size - local_M = padded_M // self.world_size - - grid_m = (M + 127) // 128 - # Avoid padding small odd grids; 2-CTA wins consistently for M >= 1024. - cta_group = 2 if M >= 1024 or grid_m % 2 == 0 else 1 - grid_m = (grid_m + cta_group - 1) // cta_group * cta_group - - num_tiles = grid_m * (N // 128) - num_ctas = min(num_tiles, self.num_sms) - num_ctas = num_ctas // cta_group * cta_group - assert self.flags.numel() >= num_tiles + num_ctas - - output = None - if not self.all_reduce: - output = torch.empty((local_M, N), dtype=torch.bfloat16, device=self.device) - # The kernel never touches rows at or beyond M. Callers that hand - # in an unpadded M (DeepSeek-V4.1 slices the SP all-gather back to - # the real token count) expect the same zero padding rows that - # ``sp_reduce_scatter`` produces, so clear them here. This is a - # no-op whenever M is already a multiple of the TP size. - valid_rows = min(max(M - self.rank * local_M, 0), local_M) - if valid_rows < local_M: - output[valid_rows:].zero_() - compiled = Sm100GemmRsAr.compile( - self.rank, - self.world_size, - cta_group, - self.all_reduce, - dtype=dtype, - ) - compiled( - x, - w, - x_sf, - w_sf, - self.partial[:padded_M], - self.partial_mc_ptr, - output, - self.flags, - self.flags_mc_ptr, - self.peer_flag_ptr, - num_ctas, - ) - if self.all_reduce: - # AttnRes may retain output past the next workspace reuse. - # A future kernel could overlap this copy using an extra warp or - # the communication warp. - return self.partial[:M].clone() - assert output is not None - return output - - -_gemm_rs_ar: GemmRsAr | None = None - - -def init_gemm_rs_ar(max_M: int, N: int, *, all_reduce: bool = False) -> None: - """Collectively initialize the process-wide, mode-bound GEMM-RS/AR state.""" - global _gemm_rs_ar - if _gemm_rs_ar is not None: - if _gemm_rs_ar.all_reduce != all_reduce: - current = "AR" if _gemm_rs_ar.all_reduce else "RS" - requested = "AR" if all_reduce else "RS" - raise RuntimeError( - f"GEMM-RS/AR is already initialized for {current}; " - f"a worker cannot reinitialize it for {requested}" - ) - assert _gemm_rs_ar.max_M >= max_M and _gemm_rs_ar.N == N - return - _gemm_rs_ar = GemmRsAr(max_M=max_M, N=N, all_reduce=all_reduce) - - -def maybe_init_gemm_rs_ar( - vllm_config: "VllmConfig", *, N: int, all_reduce: bool -) -> bool: - """Collectively initialize the mode-bound GEMM-RS/AR state if supported. - - Callers gate on their own feature flag first; this checks the worker - topology (SM100-family CUDA, BF16 model dtype, no ubatching, a TP size in - 2-16 that divides 128) and the NVLink multicast rendezvous. Returns whether - projections may bind to the kernel through ``get_gemm_rs_ar().can_run``. - """ - mode = "GEMM-AR" if all_reduce else "GEMM-RS" - parallel_config = vllm_config.parallel_config - tp_size = parallel_config.tensor_parallel_size - if parallel_config.use_ubatching: - reason = "ubatching is enabled" - elif vllm_config.model_config.dtype != torch.bfloat16: - reason = "the model dtype is not BF16" - elif not current_platform.is_cuda(): - reason = "the device is not CUDA" - elif not current_platform.is_device_capability_family(100): - reason = "the device is not SM100-family" - elif not 1 < tp_size <= 16: - reason = "TP size is not in the supported range 2-16" - elif 128 % tp_size != 0: - reason = "TP size does not divide 128" - elif N % 128 != 0: - reason = f"the output width {N} is not a multiple of 128" - else: - reason = None - - if reason is not None: - logger.warning_once("%s is disabled because %s.", mode, reason) - return False - - try: - init_gemm_rs_ar( - max_M=vllm_config.scheduler_config.max_num_batched_tokens, - N=N, - all_reduce=all_reduce, - ) - except RuntimeError as e: - logger.warning_once( - "%s is disabled because initialization failed: %s. This may mean " - "the TP ranks do not share one NVLink domain.", - mode, - e, - ) - return False - logger.info_once("%s is enabled.", mode) - return True - - -def warmup_gemm_rs_ar() -> int: - """Compile every reachable dispatch for the initialized GEMM-RS/AR mode.""" - # Initialization can be disabled or fail when multicast is unavailable. - if _gemm_rs_ar is None: - return 0 - - for dtype in sorted(_gemm_rs_ar.dtypes): - for cta_group in (1, 2): - Sm100GemmRsAr.compile( - _gemm_rs_ar.rank, - _gemm_rs_ar.world_size, - cta_group, - _gemm_rs_ar.all_reduce, - dtype=dtype, - ) - return 2 * len(_gemm_rs_ar.dtypes) - - -def get_gemm_rs_ar() -> GemmRsAr: - assert _gemm_rs_ar is not None, "GEMM-RS/AR is not initialized" - return _gemm_rs_ar diff --git a/vllm/model_executor/layers/fused_embed_norm.py b/vllm/model_executor/layers/fused_embed_norm.py deleted file mode 100644 index ddeb5156ae..0000000000 --- a/vllm/model_executor/layers/fused_embed_norm.py +++ /dev/null @@ -1,231 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Replicated input embedding + its fused gather/norm kernels. - -Groups the ``VLLM_REPLICATE_EMBED`` path in one place: the embedding factory, -the predicate that says whether the fusions apply, and the two Triton fusions -the full on-rank table unlocks -- - - * ``fused_embed_norm``: gather + a chained RMSNorm (e.g. the first decoder - layer's ``input_layernorm``), and - * ``fused_embed_eh_norm``: gather + pos-0 zeroing + enorm/hnorm + cat, the - embed/previous-hidden input norm for a speculative (MTP/eagle) depth layer - (the replicated-table analogue of the model-local ``fused_eh_norm``, which - takes precomputed embeds). - -Self-contained (no model-local imports) so it can live under ``layers/``. -""" - -import torch - -import vllm.envs as envs -from vllm.distributed import get_tensor_model_parallel_world_size -from vllm.model_executor.layers.vocab_parallel_embedding import ( - UnquantizedEmbeddingMethod, - VocabParallelEmbedding, -) -from vllm.triton_utils import tl, triton - - -@triton.jit -def _rms_norm(x, w, eps, HIDDEN_SIZE: tl.constexpr): - x = x.to(tl.float32) - mean_sq = tl.sum(x * x, axis=0) / HIDDEN_SIZE - rrms = tl.rsqrt(mean_sq + eps) - w = w.to(tl.float32) - return (x * rrms) * w - - -def make_input_embedding( - num_embeddings: int, - embedding_dim: int, - *, - params_dtype: torch.dtype | None = None, - quant_config=None, - prefix: str = "", - tie_word_embeddings: bool = False, -) -> VocabParallelEmbedding: - """Input token embedding with an optional replicated escape hatch. - - ``VLLM_REPLICATE_EMBED=1`` builds the embedding with ``disable_tp``: the full - table lives on every rank and the lookup is a local gather with no mask and - no all-reduce, which unlocks the fused gather+norm path. The cost is a full - table per rank at TP>1 (no extra memory at TP=1, where vocab-parallel is - already unsharded). A replicated, unsharded table cannot be tied to a - vocab-parallel ``ParallelLMHead``, so tied word embeddings are rejected at - TP>1 (at TP=1 ``disable_tp`` is a no-op and tying still works). - """ - disable_tp = envs.VLLM_REPLICATE_EMBED - if disable_tp and tie_word_embeddings: - assert get_tensor_model_parallel_world_size() == 1, ( - "VLLM_REPLICATE_EMBED is unsupported with tied word embeddings " - "(the replicated table cannot tie to a vocab-parallel lm_head)" - ) - return VocabParallelEmbedding( - num_embeddings, - embedding_dim, - params_dtype=params_dtype, - quant_config=quant_config, - prefix=prefix, - disable_tp=disable_tp, - ) - - -def has_full_vocab_on_rank(embedding: torch.nn.Module) -> bool: - """Whether ``embedding.weight`` is the whole vocab as a plain [V, H] table. - - The fused gather kernels index the table directly, so they need every row - on-rank (``disable_tp``, or any TP=1 run) and an unquantized weight. - """ - return getattr(embedding, "tp_size", 0) == 1 and isinstance( - getattr(embedding, "quant_method", None), UnquantizedEmbeddingMethod - ) - - -@triton.jit -def _fused_embed_norm_kernel( - ids_ptr, # [T] token ids - table_ptr, # [V, H] embedding table (full vocab, replicated on-rank) - table_stride_0, - out_ptr, # [T, H] gathered embedding (the residual stream) - normed_ptr, # [T, H] rmsnorm(out, chain_w) (HAS_NORM only) - chain_w_ptr, # [H] next norm weight (HAS_NORM only) - eps, - H: tl.constexpr, - BLOCK: tl.constexpr, - HAS_NORM: tl.constexpr, -): - tok = tl.program_id(0).to(tl.int64) - off = tl.arange(0, BLOCK) - mask = off < H - row = tl.load(ids_ptr + tok).to(tl.int64) - x = tl.load(table_ptr + row * table_stride_0 + off, mask=mask, other=0.0) - tl.store(out_ptr + tok * H + off, x, mask=mask) - if HAS_NORM: - w = tl.load(chain_w_ptr + off, mask=mask) - y = _rms_norm(x, w, eps, H).to(normed_ptr.dtype.element_ty) - tl.store(normed_ptr + tok * H + off, y, mask=mask) - - -# Base model fusion -def fused_embed_norm( - input_ids: torch.Tensor, - embed_table: torch.Tensor, - chain_weight: torch.Tensor | None = None, - eps: float = 0.0, -) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]: - """Fused embedding row gather (``embed_table[input_ids]``). - - Requires the full vocab on-rank (replicated embedding). When - ``chain_weight`` is given, also emits ``rmsnorm(gathered, chain_weight)`` - (the first decoder layer's ``input_layernorm``) as a second output in the - same launch, so the returned pair is ``(residual, normed_input)``. Bit-exact - vs a plain gather followed by an ``RMSNorm``. - """ - assert embed_table.ndim == 2, embed_table.shape - ids = input_ids.view(-1) - (t,) = ids.shape - h = embed_table.shape[1] - if chain_weight is not None: - assert chain_weight.shape == (h,), (chain_weight.shape, h) - out = torch.empty((t, h), dtype=embed_table.dtype, device=embed_table.device) - normed = torch.empty_like(out) if chain_weight is not None else None - if t > 0: - block = triton.next_power_of_2(h) - _fused_embed_norm_kernel[(t,)]( - ids, - embed_table, - embed_table.stride(0), - out, - normed if normed is not None else out, - chain_weight if chain_weight is not None else embed_table, - eps, - h, - block, - HAS_NORM=chain_weight is not None, - num_warps=min(32, max(4, block // 512)), - ) - if normed is not None: - return out, normed - return out - - -@triton.jit -def _fused_embed_eh_norm_kernel( - pos_ptr, - ids_ptr, # [T] token ids - table_ptr, # [V, H] embedding table (full vocab, replicated on-rank) - table_stride, - prev_ptr, # [T, H] previous-step hidden - prev_stride, - enorm_w_ptr, - hnorm_w_ptr, - eps, - out_ptr, # [T, 2H] - out_stride, - H: tl.constexpr, - BLOCK: tl.constexpr, -): - """MTP input fusion with a folded embedding gather: gather - ``table[ids]``, zero it at position 0, RMSNorm(embed) with enorm and - RMSNorm(prev_hidden) with hnorm, written side-by-side into ``out`` ([N, 2H]) - ready for the eh_proj GEMM. Replaces embedding lookup + where + 2x RMSNorm + - cat. Requires the full table on-rank (replicated embedding).""" - tok = tl.program_id(0) - off = tl.arange(0, BLOCK) - mask = off < H - - pos = tl.load(pos_ptr + tok) - row = tl.load(ids_ptr + tok).to(tl.int64) - e = tl.load(table_ptr + row * table_stride + off, mask=mask, other=0.0) - e = tl.where(pos == 0, 0.0, e.to(tl.float32)) - ew = tl.load(enorm_w_ptr + off, mask=mask) - e_normed = _rms_norm(e, ew, eps, H) - tl.store(out_ptr + tok * out_stride + off, e_normed, mask=mask) - - p = tl.load(prev_ptr + tok * prev_stride + off, mask=mask, other=0.0) - hw = tl.load(hnorm_w_ptr + off, mask=mask) - p_normed = _rms_norm(p, hw, eps, H) - tl.store(out_ptr + tok * out_stride + H + off, p_normed, mask=mask) - - -# MTP fusion -def fused_embed_eh_norm( - positions: torch.Tensor, - input_ids: torch.Tensor, - embed_table: torch.Tensor, - previous_hidden: torch.Tensor, - enorm_w: torch.Tensor, - hnorm_w: torch.Tensor, - eps: float, -) -> torch.Tensor: - """Fused ``cat([enorm(masked embed_table[ids]), hnorm(prev_hidden)])`` -> [N, 2H]. - - Folds the embedding row gather into the MTP eh-norm launch; requires the full - table on-rank (replicated embedding). Bit-exact vs gathering ``embed_table[ - input_ids]`` and passing it to the model-local ``fused_eh_norm``. - """ - assert previous_hidden.ndim == 2 and embed_table.ndim == 2 - n, h = previous_hidden.shape - assert positions.shape == (n,) and input_ids.view(-1).shape == (n,) - assert embed_table.shape[1] == h, (embed_table.shape, h) - assert enorm_w.shape == (h,) and hnorm_w.shape == (h,) - out = torch.empty( - n, 2 * h, dtype=previous_hidden.dtype, device=previous_hidden.device - ) - _fused_embed_eh_norm_kernel[(n,)]( - positions, - input_ids, - embed_table, - embed_table.stride(0), - previous_hidden, - previous_hidden.stride(0), - enorm_w, - hnorm_w, - eps, - out, - out.stride(0), - h, - triton.next_power_of_2(h), - ) - return out diff --git a/vllm/model_executor/layers/fused_moe/fused_flydsl_moe.py b/vllm/model_executor/layers/fused_moe/fused_flydsl_moe.py deleted file mode 100644 index 98a2577967..0000000000 --- a/vllm/model_executor/layers/fused_moe/fused_flydsl_moe.py +++ /dev/null @@ -1,391 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Fused MoE Triton kernels.""" - -import functools -import json -import os - -import torch - -from vllm.logger import init_logger -from vllm.utils.platform_utils import get_device_name_as_file_name -from vllm.utils.torch_utils import direct_register_custom_op - -logger = init_logger(__name__) - -# Valid tiles for the aiter a16w-mix int4 kernels (aiter#4646): stage1 requires -# tile_n in {64,128} and tile_k in {128,256} (k_wave=1); stage2 requires tile_n2==128 -# and tile_k2 in {128,256}. The pre-#4646 defaults (tile_k=512/64, tile_n2=256) are no -# longer registered and make the port emit an empty-fragment kernel. -_FLYDSL_MOE_DEFAULT_CONFIG = { - 1: {"tile_m": 16, "tile_n": 64, "tile_k": 128, "tile_n2": 128, "tile_k2": 128}, - 2: {"tile_m": 16, "tile_n": 64, "tile_k": 128, "tile_n2": 128, "tile_k2": 128}, - 4: {"tile_m": 16, "tile_n": 64, "tile_k": 128, "tile_n2": 128, "tile_k2": 128}, - 8: {"tile_m": 16, "tile_n": 64, "tile_k": 128, "tile_n2": 128, "tile_k2": 128}, - 16: {"tile_m": 16, "tile_n": 64, "tile_k": 128, "tile_n2": 128, "tile_k2": 128}, - 24: {"tile_m": 16, "tile_n": 64, "tile_k": 128, "tile_n2": 128, "tile_k2": 128}, - 32: {"tile_m": 16, "tile_n": 64, "tile_k": 128, "tile_n2": 128, "tile_k2": 128}, - 48: {"tile_m": 16, "tile_n": 64, "tile_k": 128, "tile_n2": 128, "tile_k2": 128}, - 64: {"tile_m": 16, "tile_n": 64, "tile_k": 128, "tile_n2": 128, "tile_k2": 128}, - 128: {"tile_m": 16, "tile_n": 64, "tile_k": 128, "tile_n2": 128, "tile_k2": 128}, - 256: {"tile_m": 16, "tile_n": 128, "tile_k": 128, "tile_n2": 128, "tile_k2": 128}, - 512: {"tile_m": 16, "tile_n": 64, "tile_k": 128, "tile_n2": 128, "tile_k2": 128}, - 1024: {"tile_m": 32, "tile_n": 64, "tile_k": 128, "tile_n2": 128, "tile_k2": 128}, - 2048: {"tile_m": 64, "tile_n": 64, "tile_k": 128, "tile_n2": 128, "tile_k2": 128}, - 4096: {"tile_m": 32, "tile_n": 64, "tile_k": 128, "tile_n2": 128, "tile_k2": 128}, - 8192: {"tile_m": 64, "tile_n": 64, "tile_k": 128, "tile_n2": 128, "tile_k2": 128}, -} - -# aiter#4646's a16w-mix int4 kernels only register a fixed tile set. Tiles from before -# the bump (default config or tuned JSONs) carry tile_k=512/64 and tile_n2=256, which -# emit an empty-fragment kernel (IndexError at emit / wrong output). Snap any resolved -# tiles to the nearest registered value so a stale tuned config can't crash the kernel. -_FLYDSL_INT4_TILE_M = (16, 32, 64, 128) -_FLYDSL_INT4_TILE_N1 = (64, 128) -_FLYDSL_INT4_TILE_K = (128, 256) - - -def _snap_flydsl_int4_tiles(tile_m, tile_n, tile_k, tile_n2, tile_k2): - def _nearest(v, allowed): - return min(allowed, key=lambda a: (abs(a - v), a)) - - snapped = ( - _nearest(tile_m, _FLYDSL_INT4_TILE_M), - _nearest(tile_n, _FLYDSL_INT4_TILE_N1), - _nearest(tile_k, _FLYDSL_INT4_TILE_K), - 128, # stage2 N tile is only registered at 128 - _nearest(tile_k2, _FLYDSL_INT4_TILE_K), - ) - if snapped != (tile_m, tile_n, tile_k, tile_n2, tile_k2): - logger.warning_once( - "FlyDSL int4 MoE tiles %s are not registered post-aiter#4646; " - "using %s. Retune the config for the a16w-mix kernel.", - (tile_m, tile_n, tile_k, tile_n2, tile_k2), - snapped, - scope="local", - ) - return snapped - - -def moe_sorting( - topk_ids: torch.Tensor, - topk_weights: torch.Tensor, - *, - num_experts: int, - model_dim: int, - block_m: int, -): - # Imported lazily so aiter's import-time sort-backend env vars are read - # when this is called, not whenever this module happens to be imported. - from aiter.fused_moe import moe_sorting as aiter_moe_sorting - - topk_ids_i32 = topk_ids.to(torch.int32) - topk_w_f32 = topk_weights.to(torch.float32) - sorted_ids, sorted_w, sorted_expert_ids, num_valid_ids, _moe_buf = ( - aiter_moe_sorting( - topk_ids_i32, - topk_w_f32, - num_experts, - model_dim, - torch.float16, - block_m, - ) - ) - if num_valid_ids.numel() > 1: - num_valid_ids = num_valid_ids[:1].contiguous() - return sorted_ids, sorted_w, sorted_expert_ids, num_valid_ids - - -def build_routing_buffers( - *, - topk_ids: torch.Tensor, - topk_weights: torch.Tensor, - num_experts: int, - model_dim: int, - tile_m: int, -): - res = moe_sorting( - topk_ids, - topk_weights, - num_experts=num_experts, - model_dim=model_dim, - block_m=tile_m, - ) - if res is None: - raise RuntimeError( - "aiter moe_sorting failed/unavailable; cannot build routing buffers." - ) - sorted_token_ids, sorted_weights, sorted_expert_ids, num_valid_ids = res - - sorted_token_ids = sorted_token_ids.contiguous() - sorted_weights = sorted_weights.contiguous() - sorted_expert_ids = sorted_expert_ids.contiguous() - sorted_size = int(sorted_token_ids.numel()) - blocks = int(sorted_expert_ids.numel()) - return ( - sorted_token_ids, - sorted_weights, - sorted_expert_ids, - num_valid_ids, - sorted_size, - blocks, - ) - - -@functools.lru_cache -def try_get_optimal_config(num_experts, inter_dim): - device_name = get_device_name_as_file_name() - json_file_name = ( - f"E={num_experts},N={inter_dim},device_name={device_name}," - "dtype=int4_w4a16,backend=flydsl.json" - ) - config_file_path = os.path.join( - os.path.dirname(os.path.realpath(__file__)), "configs", json_file_name - ) - if os.path.exists(config_file_path): - with open(config_file_path) as f: - logger.info_once( - "Using tuned FlyDSL MoE config from %s", - config_file_path, - scope="global", - ) - tuned_config = json.load(f) - return {int(key): val for key, val in tuned_config.items()} - - logger.warning_once( - "Using default FlyDSL MoE config. Performance might be sub-optimal! " - "Config file not found at %s", - config_file_path, - scope="local", - ) - return _FLYDSL_MOE_DEFAULT_CONFIG - - -def fused_flydsl_moe_impl( - hidden_states: torch.Tensor, - w1: torch.Tensor, - w2: torch.Tensor, - num_experts: int, - inter_dim: int, - topk_weights: torch.Tensor, - topk_ids: torch.Tensor, - w1_scale: torch.Tensor | None = None, - w2_scale: torch.Tensor | None = None, - topk: int = 8, - group_size: int = 32, - doweight_stage1: bool = False, - in_dtype: str = "int4_bf16", - out_dtype: str = "bf16", - scale_is_bf16: bool = True, - tile_m: int | None = None, - tile_n: int | None = None, - tile_k: int | None = None, - tile_n2: int | None = None, - tile_k2: int | None = None, -) -> torch.Tensor: - # Imported lazily -- see the comment in moe_sorting() above. - from aiter.ops.flydsl.kernels.moe_2stage_a16wmix import ( - flydsl_a16w4_gemm1, - flydsl_a16w4_gemm2, - ) - - device = hidden_states.device - tokens = hidden_states.shape[0] - model_dim = hidden_states.shape[1] - - tuned_config = {} - if tile_m and tile_n and tile_k and tile_n2 and tile_k2: - tuned_config["tile_m"] = tile_m - tuned_config["tile_n"] = tile_n - tuned_config["tile_k"] = tile_k - tuned_config["tile_n2"] = tile_n2 - tuned_config["tile_k2"] = tile_k2 - else: - tuned_config = try_get_optimal_config(num_experts, inter_dim) - tuned_config = tuned_config[ - min(tuned_config.keys(), key=lambda x: abs(x - tokens)) - ] - out_torch_dtype = torch.bfloat16 if out_dtype == "bf16" else torch.float16 - - tile_m = tuned_config["tile_m"] - tile_n = tuned_config["tile_n"] - tile_k = tuned_config["tile_k"] - tile_n2 = tuned_config["tile_n2"] - tile_k2 = tuned_config["tile_k2"] - tile_m, tile_n, tile_k, tile_n2, tile_k2 = _snap_flydsl_int4_tiles( - tile_m, tile_n, tile_k, tile_n2, tile_k2 - ) - - routing = build_routing_buffers( - topk_ids=topk_ids, - topk_weights=topk_weights, - num_experts=num_experts, - model_dim=model_dim, - tile_m=tile_m, - ) - ( - sorted_token_ids, - sorted_weights, - sorted_expert_ids, - num_valid_ids, - sorted_size, - blocks, - ) = routing - - # aiter#4646 (v0.1.20) removed compile_moe_gemm{1,2} + the flyc.compile pointer ABI - # and replaced them with the a16w-mix wrappers, which compile+launch internally - # (functools-cached) and take torch tensors directly. w_dtype="int4" is the a16wi4 - # (W4A16) path. The stage-1 intermediate now lives in sorted - # [sorted_size, inter_dim] layout, and topk weighting is applied in stage-2 via - # sorted_weights (the port has no stage-1 doweight). - sorted_weights_1d = sorted_weights.view(-1).contiguous() - empty_scale = torch.empty((0,), device=device, dtype=torch.uint8) - cumsum_tensor = num_valid_ids.to(torch.int32).contiguous() - m_indices = sorted_token_ids.to(torch.int32).contiguous() - - # stage1: bf16 A x int4 W1 + SiLU -> bf16 sorted intermediate. - inter_sorted = torch.empty( - (sorted_size, inter_dim), device=device, dtype=torch.bfloat16 - ) - w1_scale_u8 = ( - w1_scale.view(torch.uint8).contiguous().view(-1) - if w1_scale is not None - else empty_scale - ) - flydsl_a16w4_gemm1( - a_bf16=hidden_states.to(torch.bfloat16).contiguous(), - w1_u8=w1.view(torch.uint8).contiguous(), - w1_scale_u8=w1_scale_u8, - sorted_expert_ids=sorted_expert_ids, - cumsum_tensor=cumsum_tensor, - m_indices=m_indices, - inter_sorted_bf16=inter_sorted, - n_tokens=tokens, - NE=num_experts, - D_HIDDEN=model_dim, - D_INTER=inter_dim, - topk=topk, - tile_m=tile_m, - tile_n=tile_n, - tile_k=tile_k, - waves_per_eu=None, - act="silu", - w_dtype="int4", - w_layout="standard", - ) - - # stage2: bf16 inter x int4 W2 -> weighted reduce into [tokens, model_dim]. - out_stage2 = torch.zeros((tokens, model_dim), device=device, dtype=out_torch_dtype) - w2_scale_u8 = ( - w2_scale.view(torch.uint8).contiguous().view(-1) - if w2_scale is not None - else empty_scale - ) - flydsl_a16w4_gemm2( - inter_sorted_bf16=inter_sorted, - w2_u8=w2.view(torch.uint8).contiguous(), - w2_scale_u8=w2_scale_u8, - sorted_expert_ids=sorted_expert_ids, - cumsum_tensor=cumsum_tensor, - sorted_token_ids=sorted_token_ids, - sorted_weights=sorted_weights_1d, - flat_out=out_stage2.view(-1), - M_logical=tokens, - max_sorted=sorted_size, - NE=num_experts, - D_HIDDEN=model_dim, - D_INTER=inter_dim, - topk=topk, - tile_m=tile_m, - tile_n=tile_n2, - tile_k=tile_k2, - waves_per_eu=None, - w_dtype="int4", - ) - return out_stage2 - - -def fused_flydsl_moe_impl_fake( - hidden_states: torch.Tensor, - w1: torch.Tensor, - w2: torch.Tensor, - num_experts: int, - inter_dim: int, - topk_weights: torch.Tensor, - topk_ids: torch.Tensor, - w1_scale: torch.Tensor | None = None, - w2_scale: torch.Tensor | None = None, - topk: int = 8, - group_size: int = 32, - doweight_stage1: bool = False, - in_dtype: str = "int4_bf16", - out_dtype: str = "bf16", - scale_is_bf16: bool = True, - tile_m: int | None = None, - tile_n: int | None = None, - tile_k: int | None = None, - tile_n2: int | None = None, - tile_k2: int | None = None, -) -> torch.Tensor: - return torch.empty_like(hidden_states) - - -direct_register_custom_op( - op_name="fused_flydsl_moe_impl", - op_func=fused_flydsl_moe_impl, - fake_impl=fused_flydsl_moe_impl_fake, -) - - -def fused_flydsl_moe( - hidden_states: torch.Tensor, - w1: torch.Tensor, - w2: torch.Tensor, - num_experts: int, - inter_dim: int, - topk_weights: torch.Tensor, - topk_ids: torch.Tensor, - w1_scale: torch.Tensor | None = None, - w2_scale: torch.Tensor | None = None, - topk: int = 8, - group_size: int = 32, - doweight_stage1: bool = False, - in_dtype: str = "int4_bf16", - out_dtype: str = "bf16", - scale_is_bf16: bool = True, - config: dict | None = None, -) -> torch.Tensor: - tile_m = None - tile_n = None - tile_k = None - tile_n2 = None - tile_k2 = None - if config is not None: - tile_m = config.get("tile_m") - tile_n = config.get("tile_n") - tile_k = config.get("tile_k") - tile_n2 = config.get("tile_n2") - tile_k2 = config.get("tile_k2") - return torch.ops.vllm.fused_flydsl_moe_impl( - hidden_states=hidden_states, - w1=w1, - w2=w2, - num_experts=num_experts, - inter_dim=inter_dim, - topk_weights=topk_weights, - topk_ids=topk_ids, - w1_scale=w1_scale, - w2_scale=w2_scale, - topk=topk, - group_size=group_size, - doweight_stage1=doweight_stage1, - in_dtype=in_dtype, - out_dtype=out_dtype, - scale_is_bf16=scale_is_bf16, - tile_m=tile_m, - tile_n=tile_n, - tile_k=tile_k, - tile_n2=tile_n2, - tile_k2=tile_k2, - ) diff --git a/vllm/model_executor/layers/fused_moe/prepare_megamoe.py b/vllm/model_executor/layers/fused_moe/prepare_megamoe.py deleted file mode 100644 index 5e8d8cde9a..0000000000 --- a/vllm/model_executor/layers/fused_moe/prepare_megamoe.py +++ /dev/null @@ -1,682 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Triton input-staging kernels for DeepGEMM MegaMoE. - -Quantize hidden states to fp8 with E8M0 group scales, to NVFP4 with per-token -global scales, or copy them as BF16, and repack the routing top-k tensors into -the int64/float32 layout that the DeepGEMM MegaMoE kernels consume. -""" - -import torch - -from vllm.model_executor.layers.quantization.utils.quant_utils import ( - FP8_DTYPE, - QuantKey, - kMxfp8Dynamic, -) -from vllm.triton_utils import tl, triton - -_E8M0_SCALE_DTYPES = (torch.uint8, torch.float8_e8m0fnu) -# Inverse of the NVFP4 range: E2M1 max (6) times E4M3 max (448). -_INV_NVFP4_MAX = tl.constexpr(1.0 / 2688.0) - - -@triton.jit -def _quantize_fp8_groups( - hidden, - BLOCK_M: tl.constexpr, - BLOCK_K: tl.constexpr, - GROUP_K: tl.constexpr, - USE_UE8M0: tl.constexpr, -): - """Return fp8 values, fp32 group scales and, for UE8M0, the scale exponents.""" - num_groups: tl.constexpr = BLOCK_K // GROUP_K - hidden_groups = tl.reshape(tl.abs(hidden), [BLOCK_M, num_groups, GROUP_K]) - amax = tl.max(hidden_groups, axis=2) - amax = tl.maximum(amax, 1.0e-4) - - scale = amax / 448.0 - if USE_UE8M0: - scale_bits = scale.to(tl.uint32, bitcast=True) - scale_exp = ((scale_bits >> 23) & 0xFF) + ((scale_bits & 0x7FFFFF) != 0).to( - tl.uint32 - ) - scale_exp = tl.minimum(tl.maximum(scale_exp, 1), 254) - quant_scale = (scale_exp << 23).to(tl.float32, bitcast=True) - else: - scale_exp = scale.to(tl.uint32, bitcast=True) - quant_scale = scale - - hidden_groups = tl.reshape(hidden, [BLOCK_M, num_groups, GROUP_K]) - scaled = hidden_groups * (1.0 / quant_scale)[:, :, None] - scaled = tl.reshape(scaled, [BLOCK_M, BLOCK_K]) - return scaled.to(tl.float8e4nv), quant_scale, scale_exp - - -@triton.jit -def _pack_bytes(values, NUM_BYTES: tl.constexpr): - """Pack the last dim of ``[..., NUM_BYTES]`` byte values into int32 words.""" - offsets = tl.arange(0, NUM_BYTES) - return tl.sum(values.to(tl.uint32) << (offsets * 8), axis=-1).to(tl.int32) - - -@triton.jit -def _shared_sf_row(token_id, SHARED_BLOCK_M: tl.constexpr): - """Row of ``token_id`` in the shared-expert MN-major UTCCP scale pages.""" - m_block_id = token_id // SHARED_BLOCK_M - m_in_block = token_id % SHARED_BLOCK_M - aligned_block_m: tl.constexpr = triton.cdiv(SHARED_BLOCK_M, 128) * 128 - transposed_m = ( - (m_in_block // 128) * 128 + (m_in_block % 32) * 4 + (m_in_block % 128) // 32 - ) - return m_block_id * aligned_block_m + transposed_m - - -@triton.jit -def _store_topk( - token_id, - token_mask, - topk_ids, - topk_weights, - is_padding, - topk_idx_out, - topk_weights_out, - topk_ids_stride_m: tl.constexpr, - topk_ids_stride_k: tl.constexpr, - topk_weights_stride_m: tl.constexpr, - topk_weights_stride_k: tl.constexpr, - is_padding_stride_m: tl.constexpr, - topk_idx_stride_m: tl.constexpr, - topk_idx_stride_k: tl.constexpr, - topk_weights_out_stride_m: tl.constexpr, - topk_weights_out_stride_k: tl.constexpr, - top_k: tl.constexpr, - BLOCK_M: tl.constexpr, - BLOCK_TOPK: tl.constexpr, -) -> None: - """Repack top-k routing as int64 ids and fp32 weights; padding is unrouted.""" - topk_offsets = tl.arange(0, BLOCK_TOPK) - topk_mask = token_mask[:, None] & (topk_offsets[None, :] < top_k) - token_is_padding = tl.full((BLOCK_M,), False, tl.int1) - if is_padding is not None: - token_is_padding = tl.load( - is_padding + token_id * is_padding_stride_m, - mask=token_mask, - other=True, - ) - - ids = tl.load( - topk_ids - + token_id[:, None] * topk_ids_stride_m - + topk_offsets[None, :] * topk_ids_stride_k, - mask=topk_mask, - other=0, - ).to(tl.int64) - ids = tl.where(token_is_padding[:, None], -1, ids) - tl.store( - topk_idx_out - + token_id[:, None] * topk_idx_stride_m - + topk_offsets[None, :] * topk_idx_stride_k, - ids, - mask=topk_mask, - ) - - weights = tl.load( - topk_weights - + token_id[:, None] * topk_weights_stride_m - + topk_offsets[None, :] * topk_weights_stride_k, - mask=topk_mask, - other=0.0, - ) - weights = tl.where(token_is_padding[:, None], 0.0, weights) - tl.store( - topk_weights_out - + token_id[:, None] * topk_weights_out_stride_m - + topk_offsets[None, :] * topk_weights_out_stride_k, - weights, - mask=topk_mask, - ) - - -@triton.jit(do_not_specialize=["num_tokens"]) -def _prepare_megamoe_inputs_kernel( - hidden_states, - x_out, - x_sf, - shared_x_sf, - topk_ids, - topk_weights, - is_padding, - topk_idx_out, - topk_weights_out, - hidden_stride_m: tl.constexpr, - hidden_stride_k: tl.constexpr, - x_stride_m: tl.constexpr, - x_stride_k: tl.constexpr, - x_sf_stride_m: tl.constexpr, - x_sf_stride_k: tl.constexpr, - shared_x_sf_stride_m: tl.constexpr, - shared_x_sf_stride_k: tl.constexpr, - topk_ids_stride_m: tl.constexpr, - topk_ids_stride_k: tl.constexpr, - topk_weights_stride_m: tl.constexpr, - topk_weights_stride_k: tl.constexpr, - is_padding_stride_m: tl.constexpr, - topk_idx_stride_m: tl.constexpr, - topk_idx_stride_k: tl.constexpr, - topk_weights_out_stride_m: tl.constexpr, - topk_weights_out_stride_k: tl.constexpr, - num_tokens, - hidden_size: tl.constexpr, - top_k: tl.constexpr, - BLOCK_M: tl.constexpr, - BLOCK_K: tl.constexpr, - GROUP_K: tl.constexpr, - BLOCK_TOPK: tl.constexpr, - SHARED_BLOCK_M: tl.constexpr, - QUANTIZE: tl.constexpr, - USE_UE8M0: tl.constexpr, -) -> None: - token_id = tl.program_id(0) * BLOCK_M + tl.arange(0, BLOCK_M) - token_mask = token_id < num_tokens - k_block_id = tl.program_id(1) - - k_offsets = k_block_id * BLOCK_K + tl.arange(0, BLOCK_K) - k_mask = token_mask[:, None] & (k_offsets[None, :] < hidden_size) - hidden = tl.load( - hidden_states - + token_id[:, None] * hidden_stride_m - + k_offsets[None, :] * hidden_stride_k, - mask=k_mask, - other=0.0, - ).to(tl.float32) - - if not QUANTIZE: - tl.store( - x_out + token_id[:, None] * x_stride_m + k_offsets[None, :] * x_stride_k, - hidden.to(x_out.dtype.element_ty), - mask=k_mask, - ) - else: - num_groups: tl.constexpr = BLOCK_K // GROUP_K - fp8, quant_scale, scale_exp = _quantize_fp8_groups( - hidden, BLOCK_M, BLOCK_K, GROUP_K, USE_UE8M0 - ) - tl.store( - x_out + token_id[:, None] * x_stride_m + k_offsets[None, :] * x_stride_k, - fp8, - mask=k_mask, - ) - - if USE_UE8M0: - packed_scale = _pack_bytes(scale_exp, num_groups) - tl.store( - x_sf + token_id * x_sf_stride_m + k_block_id * x_sf_stride_k, - packed_scale, - mask=token_mask, - ) - else: - sf_offset = tl.arange(0, num_groups) - tl.store( - x_sf - + token_id[:, None] * x_sf_stride_m - + k_block_id * x_sf_stride_k - + sf_offset[None, :], - quant_scale, - mask=token_mask[:, None], - ) - - # DeepGEMM's SM100 shared-expert TMA loads require the activation scales - # in an MN-major layout whose row permutation depends on the MegaMoE - # scheduler's runtime BLOCK_M. Write that view while the packed UE8M0 scale - # is already resident, avoiding another kernel and temporary tensor. - if USE_UE8M0 and shared_x_sf is not None: - shared_row = _shared_sf_row(token_id, SHARED_BLOCK_M) - tl.store( - shared_x_sf - + shared_row * shared_x_sf_stride_m - + k_block_id * shared_x_sf_stride_k, - packed_scale, - mask=token_mask, - ) - - if k_block_id == 0: - _store_topk( - token_id, - token_mask, - topk_ids, - topk_weights, - is_padding, - topk_idx_out, - topk_weights_out, - topk_ids_stride_m, - topk_ids_stride_k, - topk_weights_stride_m, - topk_weights_stride_k, - is_padding_stride_m, - topk_idx_stride_m, - topk_idx_stride_k, - topk_weights_out_stride_m, - topk_weights_out_stride_k, - top_k, - BLOCK_M, - BLOCK_TOPK, - ) - - -@triton.jit(do_not_specialize=["num_tokens", "l2_activation_scale"]) -def _prepare_megamoe_nvfp4_inputs_kernel( - hidden_states, - x_out, - x_sf, - x_scales, - shared_x, - shared_x_sf, - topk_ids, - topk_weights, - is_padding, - topk_idx_out, - topk_weights_out, - w2_global_scale, - l2_alpha, - hidden_stride_m: tl.constexpr, - hidden_stride_k: tl.constexpr, - x_stride_m: tl.constexpr, - x_sf_stride_m: tl.constexpr, - shared_x_stride_m: tl.constexpr, - shared_x_sf_stride_m: tl.constexpr, - shared_x_sf_stride_k: tl.constexpr, - topk_ids_stride_m: tl.constexpr, - topk_ids_stride_k: tl.constexpr, - topk_weights_stride_m: tl.constexpr, - topk_weights_stride_k: tl.constexpr, - is_padding_stride_m: tl.constexpr, - topk_idx_stride_m: tl.constexpr, - topk_idx_stride_k: tl.constexpr, - topk_weights_out_stride_m: tl.constexpr, - topk_weights_out_stride_k: tl.constexpr, - num_tokens, - l2_activation_scale, - hidden_size: tl.constexpr, - top_k: tl.constexpr, - num_local_experts: tl.constexpr, - BLOCK_M: tl.constexpr, - BLOCK_K: tl.constexpr, - BLOCK_TOPK: tl.constexpr, - BLOCK_E: tl.constexpr, - SHARED_BLOCK_M: tl.constexpr, - SHARED_MXFP8: tl.constexpr, -) -> None: - if l2_alpha is not None: - experts = tl.arange(0, BLOCK_E) - expert_mask = (experts < num_local_experts) & (tl.program_id(0) == 0) - alpha = tl.load(w2_global_scale + experts, mask=expert_mask) - tl.store(l2_alpha + experts, alpha * l2_activation_scale, mask=expert_mask) - - token_id = tl.program_id(0) * BLOCK_M + tl.arange(0, BLOCK_M) - token_mask = token_id < num_tokens - k_offsets = tl.arange(0, BLOCK_K) - hidden_ptrs = hidden_states + token_id[:, None] * hidden_stride_m - - amax = tl.zeros((BLOCK_M,), tl.float32) - for k_block_id in range(hidden_size // BLOCK_K): - hidden = tl.load( - hidden_ptrs + (k_block_id * BLOCK_K + k_offsets[None, :]) * hidden_stride_k, - mask=token_mask[:, None], - other=0.0, - ).to(tl.float32) - amax = tl.maximum(amax, tl.max(tl.abs(hidden), axis=1)) - x_scale = tl.maximum(amax, 1.0e-12) * _INV_NVFP4_MAX - tl.store(x_scales + token_id, x_scale, mask=token_mask) - - for k_block_id in range(hidden_size // BLOCK_K): - k_ids = k_block_id * BLOCK_K + k_offsets - hidden = tl.load( - hidden_ptrs + k_ids[None, :] * hidden_stride_k, - mask=token_mask[:, None], - other=0.0, - ) - - # E2M1 values with one E4M3 scale per 16, relative to the token scale. - values = tl.reshape( - tl.div_rn(hidden.to(tl.float32), x_scale[:, None]), - [BLOCK_M, BLOCK_K // 16, 16], - ) - block_scale = tl.max(tl.abs(values), axis=2) * (1.0 / 6.0) - block_scale = tl.minimum(tl.maximum(block_scale, 2.0**-9), 448.0) - block_scale = block_scale.to(tl.float8e4nv) - values = tl.div_rn(values, block_scale.to(tl.float32)[:, :, None]) - magnitude = tl.abs(values) - # Round to nearest even on the grid {0, 0.5, 1, 1.5, 2, 3, 4, 6}. - code = ( - (magnitude > 0.25).to(tl.int32) - + (magnitude >= 0.75).to(tl.int32) - + (magnitude > 1.25).to(tl.int32) - + (magnitude >= 1.75).to(tl.int32) - + (magnitude > 2.5).to(tl.int32) - + (magnitude >= 3.5).to(tl.int32) - + (magnitude > 5.0).to(tl.int32) - ) - code = tl.where((values < 0) & (code != 0), code | 8, code) - low, high = tl.split(tl.reshape(code, [BLOCK_M, BLOCK_K // 2, 2])) - packed_offsets = k_block_id * (BLOCK_K // 2) + tl.arange(0, BLOCK_K // 2) - tl.store( - x_out + token_id[:, None] * x_stride_m + packed_offsets[None, :], - (low | (high << 4)).to(tl.uint8), - mask=token_mask[:, None], - ) - sf_offsets = k_block_id * (BLOCK_K // 64) + tl.arange(0, BLOCK_K // 64) - sf_bytes = tl.reshape( - block_scale.to(tl.uint8, bitcast=True), [BLOCK_M, BLOCK_K // 64, 4] - ) - tl.store( - x_sf + token_id[:, None] * x_sf_stride_m + sf_offsets[None, :], - _pack_bytes(sf_bytes, 4), - mask=token_mask[:, None], - ) - - if shared_x is not None: - shared_ptrs = shared_x + token_id[:, None] * shared_x_stride_m - if SHARED_MXFP8: - fp8, _, scale_exp = _quantize_fp8_groups( - hidden.to(tl.float32), BLOCK_M, BLOCK_K, 32, True - ) - tl.store(shared_ptrs + k_ids[None, :], fp8, mask=token_mask[:, None]) - tl.store( - shared_x_sf - + _shared_sf_row(token_id, SHARED_BLOCK_M) * shared_x_sf_stride_m - + k_block_id * shared_x_sf_stride_k, - _pack_bytes(scale_exp, BLOCK_K // 32), - mask=token_mask, - ) - else: - tl.store(shared_ptrs + k_ids[None, :], hidden, mask=token_mask[:, None]) - - _store_topk( - token_id, - token_mask, - topk_ids, - topk_weights, - is_padding, - topk_idx_out, - topk_weights_out, - topk_ids_stride_m, - topk_ids_stride_k, - topk_weights_stride_m, - topk_weights_stride_k, - is_padding_stride_m, - topk_idx_stride_m, - topk_idx_stride_k, - topk_weights_out_stride_m, - topk_weights_out_stride_k, - top_k, - BLOCK_M, - BLOCK_TOPK, - ) - - -def _resolve_hidden_quant(hidden_quant: QuantKey, block_k: int) -> tuple[int, bool]: - """Map a hidden-state ``QuantKey`` to the kernel's ``(GROUP_K, USE_UE8M0)``.""" - scale = hidden_quant.scale - group_shape = scale.group_shape - if ( - hidden_quant.dtype != FP8_DTYPE - or not hidden_quant.symmetric - or hidden_quant.scale2 is not None - or scale.static - or not group_shape.is_per_group() - or block_k % group_shape.col != 0 - or scale.dtype not in (*_E8M0_SCALE_DTYPES, torch.float32) - ): - raise ValueError( - "MegaMoE input staging requires symmetric dynamic fp8 " - f"per-group quantization with a group size dividing {block_k} and " - f"E8M0 or fp32 scales, got {hidden_quant}." - ) - return group_shape.col, scale.dtype in _E8M0_SCALE_DTYPES - - -def prepare_megamoe_inputs( - hidden_states: torch.Tensor, - topk_weights: torch.Tensor, - topk_ids: torch.Tensor, - x: torch.Tensor, - x_sf: torch.Tensor | None, - topk_idx_out: torch.Tensor, - topk_weights_out: torch.Tensor, - is_padding: torch.Tensor | None = None, - shared_x_sf: torch.Tensor | None = None, - shared_block_m: int | None = None, - hidden_quant: QuantKey | None = kMxfp8Dynamic, -) -> None: - """Quantize hidden states and repack top-k routing for DeepGEMM MegaMoE. - - Args: - hidden_states: Input activations of shape ``[num_tokens, hidden_dim]``. - topk_weights: Router top-k weights of shape ``[num_tokens, top_k]``. - topk_ids: Router top-k expert ids of shape ``[num_tokens, top_k]``. - x: Output buffer for the dispatched hidden states. - x_sf: Output buffer for the hidden-state scale factors; ``None`` when - ``hidden_quant`` is ``None``. - topk_idx_out: Output buffer for the repacked top-k expert ids. - topk_weights_out: Output buffer for the repacked top-k weights. - is_padding: Optional per-token mask; padded tokens are not routed. - shared_x_sf: Optional output buffer for shared-expert scale factors. - Must be given together with ``shared_block_m``. - shared_block_m: Block M used to lay out ``shared_x_sf``. - hidden_quant: Hidden-state quantization scheme. E8M0 scales are packed - four per int32 into ``x_sf``; fp32 scales are stored directly. - ``None`` copies the hidden states into ``x`` unquantized. - - """ - num_tokens, hidden_size = hidden_states.shape - if num_tokens == 0: - return - if hidden_size % 128 != 0: - raise ValueError( - "MegaMoE input staging requires hidden_size to be a multiple of 128." - ) - block_k = 128 - quantize = hidden_quant is not None - if quantize != (x_sf is not None): - raise ValueError("MegaMoE input staging needs x_sf iff hidden_quant is set.") - group_k, use_ue8m0 = ( - (block_k, False) - if hidden_quant is None - else _resolve_hidden_quant(hidden_quant, block_k) - ) - top_k = topk_ids.shape[1] - if topk_weights.shape != topk_ids.shape: - raise ValueError( - "MegaMoE input staging requires topk_weights and " - "topk_ids to have the same shape." - ) - if (shared_x_sf is None) != (shared_block_m is None): - raise ValueError( - "MegaMoE shared input staging requires both shared_x_sf and shared_block_m." - ) - if shared_x_sf is not None and not use_ue8m0: - raise ValueError( - "MegaMoE shared input staging currently requires " - "UE8M0-packed hidden scales." - ) - if shared_x_sf is not None: - assert shared_block_m is not None - if shared_block_m <= 0: - raise ValueError("MegaMoE shared_block_m must be positive.") - expected_sf_k = hidden_size // 128 - if shared_x_sf.ndim != 2 or shared_x_sf.shape[1] != expected_sf_k: - raise ValueError( - "MegaMoE shared_x_sf must have shape " - f"(*, {expected_sf_k}), got {tuple(shared_x_sf.shape)}." - ) - aligned_block_m = triton.cdiv(shared_block_m, 128) * 128 - required_rows = triton.cdiv(num_tokens, shared_block_m) * aligned_block_m - if shared_x_sf.shape[0] < required_rows: - raise ValueError( - "MegaMoE shared_x_sf has insufficient rows: requires " - f"{required_rows}, got {shared_x_sf.shape[0]}." - ) - - # On GB200, eight-row tiles win from 64 tokens; keep smaller batches untiled. - block_m = 8 if num_tokens >= 64 else 1 - grid = (triton.cdiv(num_tokens, block_m), triton.cdiv(hidden_size, block_k)) - block_topk = triton.next_power_of_2(top_k) - padding_stride_m = is_padding.stride(0) if is_padding is not None else 0 - _prepare_megamoe_inputs_kernel[grid]( - hidden_states, - x, - x_sf, - shared_x_sf, - topk_ids, - topk_weights, - is_padding, - topk_idx_out, - topk_weights_out, - hidden_states.stride(0), - hidden_states.stride(1), - x.stride(0), - x.stride(1), - x_sf.stride(0) if x_sf is not None else 0, - x_sf.stride(1) if x_sf is not None else 0, - shared_x_sf.stride(0) if shared_x_sf is not None else 0, - shared_x_sf.stride(1) if shared_x_sf is not None else 0, - topk_ids.stride(0), - topk_ids.stride(1), - topk_weights.stride(0), - topk_weights.stride(1), - padding_stride_m, - topk_idx_out.stride(0), - topk_idx_out.stride(1), - topk_weights_out.stride(0), - topk_weights_out.stride(1), - num_tokens, - hidden_size, - top_k, - BLOCK_M=block_m, - BLOCK_K=block_k, - GROUP_K=group_k, - BLOCK_TOPK=block_topk, - SHARED_BLOCK_M=shared_block_m or 1, - QUANTIZE=quantize, - USE_UE8M0=use_ue8m0, - num_warps=4, - ) - - -def prepare_megamoe_nvfp4_inputs( - hidden_states: torch.Tensor, - topk_weights: torch.Tensor, - topk_ids: torch.Tensor, - x: torch.Tensor, - x_sf: torch.Tensor, - x_scales: torch.Tensor, - topk_idx_out: torch.Tensor, - topk_weights_out: torch.Tensor, - is_padding: torch.Tensor | None = None, - shared_x: torch.Tensor | None = None, - shared_x_sf: torch.Tensor | None = None, - shared_block_m: int | None = None, - w2_global_scale: torch.Tensor | None = None, - l2_alpha: torch.Tensor | None = None, - l2_activation_scale: float = 1.0, -) -> None: - """Stage DeepGEMM NVFP4 MegaMoE inputs (``kNvfp4DynamicToken``). - - Each token is scaled by its own fp32 ``x_scales`` (row amax / 2688) and - quantized to E2M1 with one E4M3 scale per 16 values. - - Args: - hidden_states: BF16 activations of shape ``[num_tokens, hidden_dim]``. - topk_weights: Router top-k weights of shape ``[num_tokens, top_k]``. - topk_ids: Router top-k expert ids of shape ``[num_tokens, top_k]``. - x: Packed E2M1 output, ``[num_tokens, hidden_dim // 2]``. - x_sf: E4M3 scales packed four per int32, ``[num_tokens, hidden_dim // 64]``. - x_scales: Per-token fp32 dequantization scales, ``[num_tokens]``. - topk_idx_out: Output buffer for the repacked top-k expert ids. - topk_weights_out: Output buffer for the repacked top-k weights. - is_padding: Optional per-token mask; padded tokens are not routed. - shared_x: Optional shared-expert input, fp8 (MXFP8 with ``shared_x_sf``) - or a BF16 copy of the hidden states. - shared_x_sf: UE8M0 scales of an fp8 ``shared_x`` in the shared-expert - TMA layout for ``shared_block_m``. - shared_block_m: Block M used to lay out ``shared_x_sf``. - w2_global_scale: Per-local-expert L2 weight global scales. - l2_alpha: Output ``w2_global_scale * l2_activation_scale``, the L2 - dequantization scale of each local expert. Written even without - local tokens, as other ranks' tokens still reach these experts. - l2_activation_scale: Global scale the kernel quantizes L2 inputs with. - - """ - num_tokens, hidden_size = hidden_states.shape - if hidden_size % 128 != 0: - raise ValueError( - "MegaMoE input staging requires hidden_size to be a multiple of 128." - ) - if topk_weights.shape != topk_ids.shape: - raise ValueError( - "MegaMoE input staging requires topk_weights and " - "topk_ids to have the same shape." - ) - shared_mxfp8 = shared_x is not None and shared_x.dtype == FP8_DTYPE - if shared_mxfp8 != (shared_x_sf is not None) or (shared_x_sf is None) != ( - shared_block_m is None - ): - raise ValueError( - "MegaMoE NVFP4 staging needs shared_x_sf and shared_block_m exactly " - "for an fp8 shared_x." - ) - if (w2_global_scale is None) != (l2_alpha is None): - raise ValueError("MegaMoE NVFP4 staging needs w2_global_scale with l2_alpha.") - for name, tensor in (("x", x), ("x_sf", x_sf), ("shared_x", shared_x)): - if tensor is not None and tensor.stride(-1) != 1: - raise ValueError(f"MegaMoE NVFP4 staging requires a row-major {name}.") - - block_m = 8 if num_tokens >= 64 else 1 - top_k = topk_ids.shape[1] - num_local_experts = 0 if l2_alpha is None else l2_alpha.numel() - # One program even without tokens, to write l2_alpha. - grid = (max(triton.cdiv(num_tokens, block_m), 1),) - _prepare_megamoe_nvfp4_inputs_kernel[grid]( - hidden_states, - x.view(torch.uint8), - x_sf, - x_scales, - shared_x, - shared_x_sf, - topk_ids, - topk_weights, - is_padding, - topk_idx_out, - topk_weights_out, - w2_global_scale, - l2_alpha, - hidden_states.stride(0), - hidden_states.stride(1), - x.stride(0), - x_sf.stride(0), - shared_x.stride(0) if shared_x is not None else 0, - shared_x_sf.stride(0) if shared_x_sf is not None else 0, - shared_x_sf.stride(1) if shared_x_sf is not None else 0, - topk_ids.stride(0), - topk_ids.stride(1), - topk_weights.stride(0), - topk_weights.stride(1), - is_padding.stride(0) if is_padding is not None else 0, - topk_idx_out.stride(0), - topk_idx_out.stride(1), - topk_weights_out.stride(0), - topk_weights_out.stride(1), - num_tokens, - l2_activation_scale, - hidden_size, - top_k, - num_local_experts, - BLOCK_M=block_m, - BLOCK_K=128, - BLOCK_TOPK=triton.next_power_of_2(top_k), - BLOCK_E=triton.next_power_of_2(max(num_local_experts, 1)), - SHARED_BLOCK_M=shared_block_m or 1, - SHARED_MXFP8=shared_mxfp8, - num_warps=4, - ) diff --git a/vllm/model_executor/layers/fusion/relu2_fp8_quant.py b/vllm/model_executor/layers/fusion/relu2_fp8_quant.py deleted file mode 100644 index 4dc0fed4cc..0000000000 --- a/vllm/model_executor/layers/fusion/relu2_fp8_quant.py +++ /dev/null @@ -1,76 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -import torch - -from vllm.model_executor.layers.fusion.quant_activation import QuantizedActivation -from vllm.model_executor.layers.linear import LinearBase -from vllm.model_executor.layers.quantization.utils.quant_utils import ( - get_fp8_min_max, - kFp8StaticTensorSym, -) -from vllm.platforms import current_platform -from vllm.triton_utils import tl, triton - -_FP8_MIN, _FP8_MAX = get_fp8_min_max() - - -@triton.jit -def _relu_squared_static_fp8_quant_kernel( - x_ptr, - scale_ptr, - output_ptr, - num_elements, - FP8_MIN: tl.constexpr, - FP8_MAX: tl.constexpr, - BLOCK_SIZE: tl.constexpr, -): - offsets = tl.program_id(0) * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE) - mask = offsets < num_elements - x = tl.load(x_ptr + offsets, mask=mask, other=0.0).to(tl.float32) - input_is_nan = x != x - relu = tl.maximum(x, 0.0) - # Preserve the standalone ReLU2 kernel's BF16 output boundary. - activated = (relu * relu).to(tl.bfloat16).to(tl.float32) - # Match O2's Inductor-compiled QuantFP8 reciprocal-then-multiply. - quantized = activated * (1.0 / tl.load(scale_ptr)) - quantized = tl.maximum(tl.minimum(quantized, FP8_MAX), FP8_MIN) - # Match the O2 native chain by preserving NaN through the FP8 conversion. - quantized = tl.where(input_is_nan, x, quantized) - tl.store(output_ptr + offsets, quantized, mask=mask) - - -def relu_squared_static_fp8_quant( - x: torch.Tensor, linear: LinearBase -) -> QuantizedActivation: - """BF16 ReLU2 followed by static per-tensor FP8 quantization.""" - assert x.dtype == torch.bfloat16 - assert x.is_contiguous() - scale = linear.input_scale - assert scale.dtype == torch.float32 - assert x.device == scale.device - assert scale.numel() == 1 - - output = torch.empty_like(x, dtype=current_platform.fp8_dtype()) - if x.numel() != 0: - block_size = min(triton.next_power_of_2(x.shape[-1]), 2048) - num_warps = min(max(block_size // 256, 1), 4) - grid = lambda meta: (triton.cdiv(x.numel(), meta["BLOCK_SIZE"]),) - _relu_squared_static_fp8_quant_kernel[grid]( - x, - scale, - output, - x.numel(), - FP8_MIN=_FP8_MIN, - FP8_MAX=_FP8_MAX, - BLOCK_SIZE=block_size, - num_warps=num_warps, - ) - - return QuantizedActivation( - data=output, - scale=scale, - orig_dtype=x.dtype, - orig_shape=x.shape, - quant_key=kFp8StaticTensorSym, - ) diff --git a/vllm/model_executor/layers/indexer_topk.py b/vllm/model_executor/layers/indexer_topk.py deleted file mode 100644 index 02c4280ea0..0000000000 --- a/vllm/model_executor/layers/indexer_topk.py +++ /dev/null @@ -1,405 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Top-k kernels for the DSA sparse attention indexer.""" - -import functools - -import torch - -from vllm import _custom_ops as ops -from vllm.config import get_current_vllm_config -from vllm.platforms import current_platform -from vllm.utils.flashinfer import has_flashinfer -from vllm.v1.worker.workspace import current_workspace_manager - -RADIX_TOPK_WORKSPACE_SIZE = 1024 * 1024 - -# cooperative_topk's hard row limit: one cluster wave covers at most this many -# rows. "auto" uses cooperative_topk for every batch within the limit and falls -# back to persistent_topk past it; explicit ``cooperative`` requests are -# validated against the same bound. -AUTO_COOPERATIVE_MAX_ROWS = 64 - -# --------------------------------------------------------------------------- -# DeepSelect (vllm._deepselect_C) -# --------------------------------------------------------------------------- - -# Matches the -1 fill convention used for topk_indices_buffer elsewhere. -IDX_OOB_FILL_VALUE = -1 - -try: - import vllm._deepselect_C # noqa: F401 (registers torch.ops.deep_select) - - _HAS_DEEP_SELECT_C = True -except ImportError: - _HAS_DEEP_SELECT_C = False - - -@functools.lru_cache(maxsize=1) -def get_deep_select_stride_requirement() -> tuple[int, int]: - """Stride alignment requirement (input, output) in bytes.""" - return torch.ops.deep_select.get_alignment_requirement() - - -def is_deep_select_supported(input: torch.Tensor, topk: int) -> bool: - """Whether the kernel accepts this input (dtype/stride/topk constraints).""" - return ( - topk <= 4096 - and input.shape[1] < 2**23 - and input.dtype in (torch.float32, torch.bfloat16) - and input.stride(1) == 1 - and input.stride(0) - * input.element_size() - % get_deep_select_stride_requirement()[0] - == 0 - ) - - -def _get_empty_and_aligned_tensor( - dim0: int, dim1: int, device: torch.device, dtype: torch.dtype -) -> torch.Tensor: - """Tensor with shape (dim0, dim1) whose stride(0) is 32B-aligned.""" - output_stride_requirement = get_deep_select_stride_requirement()[1] // ( - dtype.itemsize - ) - assert output_stride_requirement > 0 - dim1_rounded = ( - (dim1 + output_stride_requirement - 1) // output_stride_requirement - ) * output_stride_requirement - return torch.empty((dim0, dim1_rounded), device=device, dtype=dtype)[:, :dim1] - - -def deep_select_topk( - input: torch.Tensor, - topk: int, - end: torch.Tensor | None = None, - output_idx: torch.Tensor | None = None, - indices_dtype: torch.dtype = torch.int32, -) -> torch.Tensor: - """Select the top-k indices per row of `input` with DeepSelect. - - Args: - input: (num_rows, vocab_size), bf16 or fp32. stride(1) must be 1 and - stride(0) must be 1024B-aligned. - topk: Number of elements to select per row; must be <= 4096. - end: Optional (num_rows,) int32 tensor with the exclusive right - boundary of each row. Rows with `end[i] < topk` get their - remaining indices filled with -1. - output_idx: Optional preallocated (num_rows, topk) output tensor whose - stride(0) is 32B-aligned (e.g. a slice of a wider buffer). - indices_dtype: Output dtype when `output_idx` is not provided. - - Returns: - The (num_rows, topk) indices tensor. - - """ - assert input.dim() == 2 and input.stride(1) == 1 - assert ( - input.stride(0) * input.element_size() % get_deep_select_stride_requirement()[0] - == 0 - ) - - num_rows = input.shape[0] - if output_idx is None: - output_idx = _get_empty_and_aligned_tensor( - num_rows, topk, input.device, indices_dtype - ) - else: - assert output_idx.dtype == indices_dtype - assert output_idx.shape[0] >= num_rows and output_idx.shape[1] >= topk - - torch.ops.deep_select.topk( - input, - topk, - None, # begin is not supported - end, - False, # sorted_value - False, # sorted_index - None, # output_value - output_idx, - None, # output_idx_offset - IDX_OOB_FILL_VALUE, - float("-inf"), # value_oob_fill_value - False, # return_value - False, # abort_when_nan_found: dummy/capture passes can feed NaN - # logits from uninitialized KV cache; the reference topk kernels do - # not trap on NaN, and trapping would abort CUDA graph capture. - ) - return output_idx - - -# --------------------------------------------------------------------------- -# Backend selection -# --------------------------------------------------------------------------- - - -@functools.cache -def get_indexer_topk(backend: str) -> "SparseIndexerTopk": - return SparseIndexerTopk(backend) - - -class SparseIndexerTopk(torch.nn.Module): - """The sparse indexer's decode top-k stage. - - Selects among the available top-k kernels (see - kernel_config.sparse_indexer_topk_backend) and runs the chosen one. - """ - - def __init__(self, backend: str | None = None) -> None: - super().__init__() - if backend is None: - backend = ( - get_current_vllm_config().kernel_config.sparse_indexer_topk_backend - ) - self._backend = backend - self._is_cuda = current_platform.is_cuda() - self._has_deep_select = self._is_cuda and ( - current_platform.is_device_capability_family(100) - ) - self._has_flashinfer_topk = has_flashinfer() - self._cooperative_capable = self._is_cuda and ( - current_platform.has_device_capability(90) - and not current_platform.is_device_capability_family(120) - ) - self._is_rocm = current_platform.is_rocm() - self._aiter_enabled = False - self._aiter_capable = False - if self._is_rocm: - from vllm._aiter_ops import rocm_aiter_ops - - self._aiter_enabled = bool(rocm_aiter_ops.is_enabled()) - self._aiter_capable = bool(rocm_aiter_ops.is_indexer_top_k_enabled()) - - def resolve_backend( - self, logits: torch.Tensor, topk_tokens: int, num_rows: int - ) -> str: - """Resolve the decode top-k implementation from the configured - backend ("auto" = the pre-existing chain, or a validated explicit - value).""" - if self._backend == "auto": - return self._resolve_auto(logits, topk_tokens, num_rows) - - failures: list[str] = [] - if self._backend == "cooperative": - failures = self._cooperative_constraints(logits, topk_tokens, num_rows) - elif self._backend == "persistent": - if not self._is_cuda: - failures.append("requires a CUDA platform") - if topk_tokens not in (512, 1024, 2048): - failures.append( - f"topk_tokens must be in (512, 1024, 2048), got {topk_tokens}" - ) - elif self._backend == "deep_select": - if not self._is_cuda: - failures.append("requires a CUDA platform") - elif not self._has_deep_select: - failures.append("requires SM100a/SM103a (10.x device family)") - elif not _HAS_DEEP_SELECT_C: - failures.append("vllm._deepselect_C could not be imported") - elif not is_deep_select_supported(logits, topk_tokens): - failures.append( - f"inputs violate DeepSelect's constraints: dtype={logits.dtype}," - f" stride={logits.stride()}, topk={topk_tokens}" - ) - elif self._backend == "aiter": - if not self._is_rocm: - failures.append("requires a ROCm platform") - elif not self._aiter_enabled: - failures.append("requires AITER (VLLM_ROCM_USE_AITER=1)") - elif not self._aiter_capable: - failures.append( - "AITER has no tuned indexer top-k kernels for this GPU arch" - ) - if topk_tokens not in (512, 1024, 2048, 4096): - failures.append( - f"topk_tokens must be in (512, 1024, 2048, 4096), got {topk_tokens}" - ) - elif self._backend == "flashinfer": - if not self._is_cuda: - failures.append("requires a CUDA platform") - if not self._has_flashinfer_topk: - failures.append( - "flashinfer.topk.top_k_ragged_transform is not importable" - ) - if logits.dtype != torch.float32 or logits.stride(1) != 1: - failures.append( - f"requires fp32 logits with stride(1) == 1, got " - f"dtype={logits.dtype}, stride={logits.stride()}" - ) - if failures: - raise RuntimeError( - f"sparse_indexer_topk_backend='{self._backend}' was requested, but: " - + "; ".join(failures) - ) - return self._backend - - def _resolve_auto( - self, logits: torch.Tensor, topk_tokens: int, num_rows: int - ) -> str: - """The priority chain: cooperative -> persistent -> per_row. - aiter/deep_select/flashinfer/torch are opt-in only. - - cooperative_topk is preferred whenever it is applicable, i.e. within - its AUTO_COOPERATIVE_MAX_ROWS row limit; larger batches go to - persistent_topk. - - aiter not auto-resolved as it does not outperform the per-row on every shape, - hence we delegate to the caller to decide when to use it. - """ - if not self._cooperative_constraints(logits, topk_tokens, num_rows): - return "cooperative" - if self._is_cuda and topk_tokens in (512, 1024, 2048): - return "persistent" - return "per_row" - - def _cooperative_constraints( - self, logits: torch.Tensor, topk_tokens: int, num_rows: int - ) -> list[str]: - """Unmet constraints of cooperative_topk (empty when applicable).""" - failures = [] - if not self._is_cuda: - failures.append("requires a CUDA platform") - if topk_tokens not in (512, 1024, 2048): - failures.append( - f"topk_tokens must be in (512, 1024, 2048), got {topk_tokens}" - ) - if num_rows > AUTO_COOPERATIVE_MAX_ROWS: - failures.append( - f"num_rows must be <= {AUTO_COOPERATIVE_MAX_ROWS}, got {num_rows}" - ) - if logits.stride(0) % 4 != 0: - failures.append( - f"logits.stride(0) must be divisible by 4, got {logits.stride(0)}" - ) - if self._is_cuda and not self._cooperative_capable: - failures.append("requires SM90+ and is not supported on the SM12x family") - return failures - - @staticmethod - def _row_ends(seq_lens: torch.Tensor, next_n: int, num_rows: int) -> torch.Tensor: - """Per-row exclusive end offsets (int32, (num_rows,)) for top-k - kernels that take ragged lengths (DeepSelect, FlashInfer, torch - reference). - - seq_lens is (B, next_n) per-row effective lens for native spec decode - and (B, 1) otherwise, in which case per-row lens are derived the same - way as the other decode top-k kernels. - """ - if seq_lens.numel() == num_rows: - row_ends = seq_lens.reshape(-1) - else: - next_n_offsets = torch.arange( - next_n, dtype=torch.int32, device=seq_lens.device - ) - row_ends = ( - (seq_lens.reshape(-1, 1) - next_n + 1 + next_n_offsets) - .clamp_(min=0) - .reshape(-1) - ) - assert row_ends.dtype == torch.int32 - return row_ends - - def forward( - self, - logits: torch.Tensor, - seq_lens: torch.Tensor, - next_n: int, - topk_indices: torch.Tensor, - topk_tokens: int, - max_seq_len: int, - ) -> None: - """Run the resolved decode top-k implementation, writing into - topk_indices (int32, -1 fill for rows shorter than topk_tokens). - """ - backend = self.resolve_backend(logits, topk_tokens, logits.shape[0]) - if backend == "deep_select": - row_ends = self._row_ends(seq_lens, next_n, logits.shape[0]) - deep_select_topk(logits, topk_tokens, end=row_ends, output_idx=topk_indices) - elif backend == "cooperative": - (topk_workspace,) = current_workspace_manager().get_simultaneous( - ((RADIX_TOPK_WORKSPACE_SIZE,), torch.uint8), - ) - torch.ops._C.cooperative_topk( - logits, - seq_lens, - topk_indices, - topk_workspace, - topk_tokens, - max_seq_len, - ) - elif backend == "persistent": - (topk_workspace,) = current_workspace_manager().get_simultaneous( - ((RADIX_TOPK_WORKSPACE_SIZE,), torch.uint8), - ) - # persistent_topk's last argument is the host-side "max seq_len - # across rows" bound (it gates the sampled_topk path and clamps - # per-row lengths), not the padded row width: logits is sized to - # max_model_len, so passing its width would incorrectly enable the - # long-row kernels on short-context batches. - torch.ops._C.persistent_topk( - logits, - seq_lens, - topk_indices, - topk_workspace, - topk_tokens, - max_seq_len, - ) - elif backend == "flashinfer": - # Deferred: importing flashinfer initializes CUDA at import time. - from flashinfer.topk import top_k_ragged_transform - - row_ends = self._row_ends(seq_lens, next_n, logits.shape[0]) - offsets = torch.zeros( - logits.shape[0], dtype=torch.int32, device=logits.device - ) - # top_k_ragged_transform selects within [0, row_ends[i]) per - # row and -1-fills past the row length. - indices = top_k_ragged_transform(logits, offsets, row_ends, topk_tokens) - topk_indices.copy_(indices) - elif backend == "torch": - # Debug reference: mask everything past each row's end, then topk. - row_ends = self._row_ends(seq_lens, next_n, logits.shape[0]) - cols = torch.arange(logits.shape[1], device=logits.device) - masked = logits.masked_fill( - cols.unsqueeze(0) >= row_ends.unsqueeze(1), float("-inf") - ) - indices = masked.topk(topk_tokens, dim=-1).indices - in_range = torch.arange(topk_tokens, device=logits.device).unsqueeze( - 0 - ) < row_ends.unsqueeze(1) - indices = torch.where(in_range, indices, -1) - topk_indices.copy_(indices) - elif backend == "aiter": - from vllm._aiter_ops import rocm_aiter_ops - - # AITER's decode kernel only implements the 1D/next_n == 1 form, - # which is equivalent once the ragged ends are expanded. - rocm_aiter_ops.indexer_top_k_decode( - logits, - 1, - self._row_ends(seq_lens, next_n, logits.shape[0]), - topk_indices, - topk_tokens, - ) - else: - assert backend == "per_row", f"unknown topk backend: {backend}" - self._per_row(logits, seq_lens, next_n, topk_indices, topk_tokens) - - @staticmethod - def _per_row( - logits: torch.Tensor, - seq_lens: torch.Tensor, - next_n: int, - topk_indices: torch.Tensor, - topk_tokens: int, - ) -> None: - ops.top_k_per_row_decode( - logits, - next_n, - seq_lens, - topk_indices, - logits.shape[0], - logits.stride(0), - logits.stride(1), - topk_tokens, - ) diff --git a/vllm/model_executor/layers/lightning_attn.py b/vllm/model_executor/layers/lightning_attn.py deleted file mode 100644 index 1df196ca4d..0000000000 --- a/vllm/model_executor/layers/lightning_attn.py +++ /dev/null @@ -1,741 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -import torch -from einops import rearrange - -from vllm.platforms import current_platform -from vllm.triton_utils import tl, triton -from vllm.v1.attention.backends.utils import PAD_SLOT_ID - - -@triton.jit -def _fwd_diag_kernel( - Q, - K, - V, - Out, - S, - b: tl.constexpr, - h: tl.constexpr, - n, - d: tl.constexpr, - e: tl.constexpr, - BLOCK: tl.constexpr, - NUM_BLOCK, - CBLOCK: tl.constexpr, -): - # This kernel computes the diagonal blocks of the attention matrix - # Each diagonal block represents attention - # where queries attend to keys in the same block - off = tl.program_id(0) - off_bh = off // NUM_BLOCK # batch-head index - off_block = off % NUM_BLOCK # block index within the sequence - off_cblock = tl.program_id(1) # sub-block index within a block - - off_h = off_bh % h # head index - - # Calculate base offsets for the current batch and head - qk_offset = off_bh * n * d - v_offset = off_bh * n * e - o_offset = off_bh * n * e - - # Calculate offsets for the current block - block_offset = off_block * BLOCK - qk_block_offset = block_offset * d - v_block_offset = block_offset * e - o_block_offset = block_offset * e - - # Calculate offsets for the current sub-block - cblock_offset = off_cblock * CBLOCK - q_cblock_offset = cblock_offset * d - o_cblock_offset = cblock_offset * e - - # Calculate pointers to the query, key, value, and output tensors - Q_block_ptr = ( - Q - + qk_offset - + qk_block_offset - + q_cblock_offset - + tl.arange(0, CBLOCK)[:, None] * d - + tl.arange(0, d)[None, :] - ) - K_trans_block_ptr = ( - K - + qk_offset - + qk_block_offset - + tl.arange(0, CBLOCK)[None, :] * d - + tl.arange(0, d)[:, None] - ) - V_block_ptr = ( - V - + v_offset - + v_block_offset - + tl.arange(0, CBLOCK)[:, None] * e - + tl.arange(0, e)[None, :] - ) - O_block_ptr = ( - Out - + o_offset - + o_block_offset - + o_cblock_offset - + tl.arange(0, CBLOCK)[:, None] * e - + tl.arange(0, e)[None, :] - ) - - # Load the decay rate for the current head - S_block_ptr = S + off_h - s = tl.load(S_block_ptr) - - i = off_cblock - q_index = tl.arange(0, CBLOCK) + i * CBLOCK - - # Load query values - q = tl.load(Q_block_ptr, mask=block_offset + q_index[:, None] < n, other=0.0).to( - tl.float32 - ) - - # Initialize output accumulator - qkv = tl.zeros([CBLOCK, e], dtype=tl.float32) - - # Process all sub-blocks up to and - # including the current one (causal attention) - for j in range(i + 1): - kv_index = tl.arange(0, CBLOCK) + j * CBLOCK - diff = q_index[:, None] - kv_index[None, :] - s_index = s * diff - # Apply causal mask: only attend to positions before the current one - s_index = tl.where(diff >= 0, -s_index, float("-inf")) - decay = tl.exp(s_index) - - # Load key and value - k_trans = tl.load( - K_trans_block_ptr, - mask=block_offset + kv_index[None, :] < n, - other=0.0, - ).to(tl.float32) - v = tl.load( - V_block_ptr, - mask=block_offset + kv_index[:, None] < n, - other=0.0, - ).to(tl.float32) - - # Compute attention scores and apply decay - qk = tl.dot(q, k_trans) * decay - - # Compute weighted values and accumulate - qkv += tl.dot(qk, v) - - # Move to the next sub-block - K_trans_block_ptr += CBLOCK * d - V_block_ptr += CBLOCK * e - - # Store the result - tl.store( - O_block_ptr, - qkv.to(O_block_ptr.dtype.element_ty), - mask=block_offset + q_index[:, None] < n, - ) - - -@triton.jit -def _fwd_kv_parallel( - K, - V, - K_decay, - KV, - b: tl.constexpr, - h: tl.constexpr, - n, - d: tl.constexpr, - e: tl.constexpr, - BLOCK: tl.constexpr, - NUM_BLOCK, - D_FBLOCK: tl.constexpr, - E_FBLOCK: tl.constexpr, - NUM_FBLOCK: tl.constexpr, - CBLOCK: tl.constexpr, - NUM_CBLOCK: tl.constexpr, -): - # This kernel computes the key-value outer - # products for each block in parallel - off_bh = tl.program_id(0) # batch-head index - off_block = tl.program_id(1) # block index - - off_h = off_bh % h # head index - - block_offset = off_block * BLOCK - - # Calculate offsets for the current block - k_block_offset = block_offset * d - v_block_offset = block_offset * e - kv_block_offset = off_block * d * e - - # Calculate base offsets for the current batch and head - k_offset = off_bh * n * d - v_offset = off_bh * n * e - kv_offset = off_bh * NUM_BLOCK * d * e - - # Calculate pointers to the key, value, and key-value tensors - K_trans_block_ptr = ( - K - + k_offset - + k_block_offset - + tl.arange(0, CBLOCK)[None, :] * d - + tl.arange(0, D_FBLOCK)[:, None] - ) - V_block_ptr = ( - V - + v_offset - + v_block_offset - + tl.arange(0, CBLOCK)[:, None] * e - + tl.arange(0, E_FBLOCK)[None, :] - ) - KV_block_ptr = ( - KV - + kv_offset - + kv_block_offset - + tl.arange(0, D_FBLOCK)[:, None] * e - + tl.arange(0, E_FBLOCK)[None, :] - ) - - # Load the decay factors for the current head and block - k_decay_ptr = K_decay + off_h * BLOCK + tl.arange(0, CBLOCK) - - kv_index = tl.arange(0, CBLOCK) - - # Initialize the key-value outer product accumulator - kv = tl.zeros([D_FBLOCK, E_FBLOCK], dtype=tl.float32) - - # Handle the last block which might be smaller than BLOCK - split_n = n - (NUM_BLOCK - 1) * BLOCK if off_block == NUM_BLOCK - 1 else BLOCK - left_shift = tl.cdiv(split_n, CBLOCK) * CBLOCK - split_n - num_blocks = min(tl.cdiv(split_n, CBLOCK), NUM_CBLOCK) - k_decay_ptr += (NUM_CBLOCK - num_blocks) * CBLOCK - - # Process all sub-blocks in the current block - for j in range(num_blocks): - left_bound = (1 - j) * left_shift - # Load key and value, handling boundary conditions - k_trans = tl.load( - K_trans_block_ptr - left_shift * d, - mask=kv_index[None, :] >= left_bound, - other=0.0, - ) - v = tl.load( - V_block_ptr - left_shift * e, - mask=kv_index[:, None] >= left_bound, - other=0.0, - ) - - # Load decay factor and compute weighted key-value outer product - k_decay = tl.load(k_decay_ptr) - - # NOTE: Need to add the extra dim here due to AMD MLIR lowering error. - # Please don't move it back until issue is resolved. - # Issue: https://github.com/ROCm/triton/issues/907 - k_decay = k_decay[None, :] - - kv += tl.dot(k_trans * k_decay, v) - - # Move to the next sub-block - K_trans_block_ptr += CBLOCK * d - V_block_ptr += CBLOCK * e - k_decay_ptr += CBLOCK - - # Store the result - tl.store(KV_block_ptr, kv.to(KV_block_ptr.dtype.element_ty)) - - -@triton.jit -def _fwd_kv_reduce( - S, - KV, - KV_HISTORY, - b: tl.constexpr, - h: tl.constexpr, - n, - d: tl.constexpr, - e: tl.constexpr, - BLOCK: tl.constexpr, - NUM_BLOCK, - D_FBLOCK: tl.constexpr, - E_FBLOCK: tl.constexpr, -): - # This kernel reduces the key-value outer products - # across blocks and updates the KV history - off_bh = tl.program_id(0) # batch-head index - off_h = off_bh % h # head index - - kv_offset = off_bh * NUM_BLOCK * d * e - - # Calculate pointer to the key-value tensor - KV_block_ptr = ( - KV - + kv_offset - + tl.arange(0, D_FBLOCK)[:, None] * e - + tl.arange(0, E_FBLOCK)[None, :] - ) - - # Load the decay rate for the current head - s_ptrs = S + off_h - s = tl.load(s_ptrs) - - # Calculate pointer to the key-value history tensor - kv_history_offset = off_bh * d * e - KV_HISTORY_block_ptr = ( - KV_HISTORY - + kv_history_offset - + tl.arange(0, D_FBLOCK)[:, None] * e - + tl.arange(0, E_FBLOCK)[None, :] - ) - - # Load the previous key-value history - kv_pre = tl.load(KV_HISTORY_block_ptr).to(tl.float32) - - # Process all blocks in reverse order to compute the prefix sum - for i in range(NUM_BLOCK): - block_size = min(n - i * BLOCK, BLOCK) - # Compute decay factor for the current block - block_decay = tl.exp(-s.to(tl.float32) * block_size) - - # Load the current key-value outer product - kv_cur = tl.load(KV_block_ptr).to(tl.float32) - # Store the previous key-value history to the current block - tl.store(KV_block_ptr, kv_pre.to(KV_block_ptr.dtype.element_ty)) - - # Update the key-value history with the current block - kv_pre = block_decay * kv_pre + kv_cur - KV_block_ptr += d * e - - # Store the updated key-value history - tl.store(KV_HISTORY_block_ptr, kv_pre) - - -@triton.jit -def _fwd_none_diag_kernel( - Q, - Out, - S, - KV, - b: tl.constexpr, - h: tl.constexpr, - n, - d: tl.constexpr, - e: tl.constexpr, - BLOCK: tl.constexpr, - NUM_BLOCK, - E_FBLOCK: tl.constexpr, - CBLOCK: tl.constexpr, - NUM_CBLOCK: tl.constexpr, -): - # This kernel computes the non-diagonal blocks of the attention matrix - # Each non-diagonal block represents attention - # where queries attend to keys in different blocks - off_bh = tl.program_id(0) # batch-head index - off_h = off_bh % h # head index - - off_nc = tl.program_id(1) - off_n = off_nc // NUM_CBLOCK # block index - off_c = off_nc % NUM_CBLOCK # sub-block index - off_e = tl.program_id(2) # output feature block index - - n_offset = off_n * BLOCK - c_offset = off_c * CBLOCK - e_offset = off_e * E_FBLOCK - block_offset = n_offset + c_offset - - # Calculate offsets for the current batch, head, and block - q_offset = off_bh * n * d + (n_offset + c_offset) * d - o_offset = off_bh * n * e + (n_offset + c_offset) * e + e_offset - kv_offset = off_bh * NUM_BLOCK * d * e + off_n * d * e + e_offset - - # Calculate pointers to the query, output, and key-value tensors - Q_block_ptr = ( - Q + q_offset + tl.arange(0, CBLOCK)[:, None] * d + tl.arange(0, d)[None, :] - ) - O_block_ptr = ( - Out - + o_offset - + tl.arange(0, CBLOCK)[:, None] * e - + tl.arange(0, E_FBLOCK)[None, :] - ) - KV_block_ptr = ( - KV + kv_offset + tl.arange(0, d)[:, None] * e + tl.arange(0, E_FBLOCK)[None, :] - ) - - # Load the decay rate for the current head - S_block_ptr = S + off_h - s = tl.load(S_block_ptr) - - c_array = tl.arange(0, CBLOCK) - - # Load the key-value outer product for the current block - kv = tl.load(KV_block_ptr).to(tl.float32) - q_index = block_offset + tl.arange(0, CBLOCK) - - # Load query values - q = tl.load(Q_block_ptr, mask=q_index[:, None] < n, other=0.0).to(tl.float32) - - # Compute decay factors for the current sub-block - q_decay = tl.exp(-s.to(tl.float32) * (off_c * CBLOCK + c_array[:, None])) - - # Compute non-diagonal attention output - qkv_none_diag = tl.dot(q, kv) * q_decay - - # Load diagonal attention output (computed by _fwd_diag_kernel) - qkv_diag = tl.load(O_block_ptr, mask=q_index[:, None] < n, other=0.0).to(tl.float32) - - # Combine diagonal and non-diagonal attention outputs - qkv = qkv_diag + qkv_none_diag - - # Store the result - tl.store( - O_block_ptr, qkv.to(O_block_ptr.dtype.element_ty), mask=q_index[:, None] < n - ) - - -class _attention(torch.autograd.Function): - @staticmethod - def forward(ctx, q, k, v, s, kv_history): - # Forward pass of the lightning attention algorithm - q = q.contiguous() - k = k.contiguous() - v = v.contiguous() - s = s.contiguous() - - # Check CUDA compute capability (Ampere+ required for flash attention - # path). Other accelerators (ROCm, XPU) rely on their own Triton - # backend support and skip this check. - if current_platform.is_cuda(): - capability = torch.cuda.get_device_capability() - if capability[0] < 8: - raise RuntimeError( - "Flash attention currently only supported", - "for compute capability >= 80", - ) - - # Get input dimensions - b, h, n, d = q.shape - e = v.shape[-1] - - # Initialize output tensor - o = torch.empty((b, h, n, e), dtype=q.dtype, device=q.device) - - # Set block sizes - BLOCK = 256 - NUM_BLOCK = triton.cdiv(n, BLOCK) - - CBLOCK = 32 - NUM_CBLOCK = BLOCK // CBLOCK - assert BLOCK % CBLOCK == 0, "BLOCK must be a multiple of CBLOCK" - - # Compute decay factors for keys - array = torch.arange(0, BLOCK, device=q.device) + 1 - k_decay = torch.exp(-s * (BLOCK - array.reshape(1, -1))) - - # Step 1: Compute diagonal blocks of attention - grid = (b * h * NUM_BLOCK, NUM_CBLOCK) - _fwd_diag_kernel[grid]( - q, - k, - v, - o, - s, - b, - h, - n, - d, - e, - BLOCK=BLOCK, - NUM_BLOCK=NUM_BLOCK, - CBLOCK=CBLOCK, - ) - - # Set feature block sizes - NUM_FBLOCK = 1 - D_FBLOCK = d // NUM_FBLOCK - assert d % NUM_FBLOCK == 0 - E_FBLOCK = e // NUM_FBLOCK - assert e % NUM_FBLOCK == 0 - - CBLOCK = 64 - NUM_CBLOCK = BLOCK // CBLOCK - assert BLOCK % CBLOCK == 0, "BLOCK must be a multiple of CBLOCK" - - # Step 2: Compute key-value outer products for each block in parallel - kv = torch.empty((b, h, NUM_BLOCK, d, e), dtype=torch.float32, device=q.device) - grid = (b * h, NUM_BLOCK) - _fwd_kv_parallel[grid]( - k, - v, - k_decay, - kv, - b, - h, - n, - d, - e, - BLOCK=BLOCK, - NUM_BLOCK=NUM_BLOCK, - D_FBLOCK=D_FBLOCK, - E_FBLOCK=E_FBLOCK, - NUM_FBLOCK=NUM_FBLOCK, - CBLOCK=CBLOCK, - NUM_CBLOCK=NUM_CBLOCK, - ) - - # Step 3: Reduce key-value outer products - # across blocks and update KV history - grid = (b * h, NUM_FBLOCK) - _fwd_kv_reduce[grid]( - s, - kv, - kv_history, - b, - h, - n, - d, - e, - BLOCK=BLOCK, - NUM_BLOCK=NUM_BLOCK, - D_FBLOCK=D_FBLOCK, - E_FBLOCK=E_FBLOCK, - ) - - # Step 4: Compute non-diagonal blocks of attention - grid = (b * h, NUM_BLOCK * NUM_CBLOCK) - _fwd_none_diag_kernel[grid]( - q, - o, - s, - kv, - b, - h, - n, - d, - e, - BLOCK=BLOCK, - NUM_BLOCK=NUM_BLOCK, - E_FBLOCK=E_FBLOCK, - CBLOCK=CBLOCK, - NUM_CBLOCK=NUM_CBLOCK, - ) - - # Save tensors for backward pass - ctx.save_for_backward(q, k, v, s, kv) - ctx.BLOCK = BLOCK - - return o, torch.cat([kv, kv_history.unsqueeze(2)], dim=2) - - -# Apply the lightning attention function -lightning_attention_ = _attention.apply - - -def lightning_attention( - q: torch.Tensor, - k: torch.Tensor, - v: torch.Tensor, - ed: torch.Tensor, - block_size: int = 256, - kv_history: torch.Tensor | None = None, -) -> tuple[torch.Tensor, torch.Tensor]: - """Apply lightning attention algorithm - to compute attention efficiently. - - Args: - q: Query tensor of shape [batch, heads, seq_len, dim] - k: Key tensor of shape [batch, heads, seq_len, dim] - v: Value tensor of shape [batch, heads, seq_len, dim_v] - ed: Decay rate tensor of shape [heads] - block_size: Size of blocks for block-sparse attention - kv_history: Optional key-value history from previous computations - - Returns: - output: Attention output - kv: Updated key-value history - - """ - d = q.shape[-1] - e = v.shape[-1] - - if ed.dim() == 1: - ed = ed.view(1, -1, 1, 1) - - # Split the computation into chunks for better parallelism - m = 128 if d >= 128 else 64 - assert d % m == 0, f"Dimension d ({d}) must be divisible by m ({m})" - arr = [m * i for i in range(d // m + 1)] - if arr[-1] != d: - arr.append(d) - n = len(arr) - output = 0 - - # Initialize or clone key-value history - if kv_history is None: - kv_history = torch.zeros( - (q.shape[0], q.shape[1], d, e), dtype=torch.float32, device=q.device - ) - else: - kv_history = kv_history.clone().contiguous() - - # Process each chunk and accumulate results - for i in range(n - 1): - s = arr[i] - e = arr[i + 1] - q1 = q[..., s:e] - k1 = k[..., s:e] - o, kv = lightning_attention_(q1, k1, v, ed, kv_history) - output = output + o - return output, kv - - -@triton.jit -def _linear_attn_decode_kernel( - q_ptr, - k_ptr, - v_ptr, - kv_cache_ptr, - slope_rate, - slot_idx, - output_ptr, - D: tl.constexpr, - qkv_b_stride, - qkv_h_stride, - cache_b_stride, - cache_h_stride, - cache_d0_stride, - cache_d1_stride, - pad_slot_id: tl.constexpr, - BLOCK_SIZE: tl.constexpr, -): - """Kernel for linear attention decoding with KV cache. - - This kernel computes attention for a single token using the KV cache. - """ - pid_b = tl.program_id(0) # batch index - pid_h = tl.program_id(1) # head index - pid_d = tl.program_id(2) # dimension block index - - # Load slot index for the current batch - slot_id = tl.load(slot_idx + pid_b).to(tl.int64) - - # Skip if slot_id is PAD_SLOT_ID (padding) - if slot_id == pad_slot_id: - return - - batch_id = pid_b - head_id = pid_h - - # Load decay rate for the current head - ratio = tl.load(slope_rate + pid_h) - - # Calculate offsets for dimensions - qk_d_offsets = tl.arange(0, D) - v_d_offsets = tl.arange(0, BLOCK_SIZE) + pid_d * BLOCK_SIZE - cache_d_offsets = ( - qk_d_offsets[:, None] * cache_d0_stride + v_d_offsets[None, :] * cache_d1_stride - ) - - # Calculate offsets for the current batch and head - q_offset = batch_id * qkv_b_stride + head_id * qkv_h_stride - k_offset = batch_id * qkv_b_stride + head_id * qkv_h_stride - v_offset = batch_id * qkv_b_stride + head_id * qkv_h_stride - - cache_offset = slot_id * cache_b_stride + head_id * cache_h_stride - - # Create masks for loading tensors - qk_mask = qk_d_offsets < D - v_mask = v_d_offsets < D - - # Load query, key, and value tensors - q = tl.load(q_ptr + q_offset + qk_d_offsets, mask=qk_mask, other=0.0) - k = tl.load(k_ptr + k_offset + qk_d_offsets, mask=qk_mask, other=0.0) - v = tl.load(v_ptr + v_offset + v_d_offsets, mask=v_mask, other=0.0) - - # Compute key-value outer product - kv_outer = k[:, None] * v[None, :] - kv_mask = qk_mask[:, None] & v_mask[None, :] - - # Apply decay to previous KV cache - ratio = tl.exp(-ratio) - kv_ptr = kv_cache_ptr + cache_offset + cache_d_offsets - kv_cache_old = tl.load(kv_ptr, mask=kv_mask, other=0.0) - kv_outer = kv_outer + ratio * kv_cache_old - - # Compute attention output - output = q[:, None].to(tl.float32) * kv_outer - output = tl.sum(output, axis=0) - - # Update KV cache and store output - tl.store(kv_ptr, kv_outer, mask=kv_mask) - tl.store(output_ptr + q_offset + v_d_offsets, output, mask=v_mask) - - -def linear_decode_forward_triton( - q: torch.Tensor, - k: torch.Tensor, - v: torch.Tensor, - kv_caches: torch.Tensor, - slope_rate: torch.Tensor, - slot_idx: torch.Tensor, - BLOCK_SIZE: int = 32, -) -> torch.Tensor: - """Perform linear attention decoding using Triton kernels. - - Args: - q: Query tensor of shape [B, H, 1, D] - k: Key tensor of shape [B, H, 1, D] - v: Value tensor of shape [B, H, 1, D] - kv_caches: Key-value cache tensor - slope_rate: Decay rate tensor - slot_idx: Slot indices for batches - BLOCK_SIZE: Size of blocks for processing - - Returns: - output: Attention output tensor - - """ - B, H, _, D = q.shape - assert k.shape == (B, H, 1, D) - assert v.shape == (B, H, 1, D) - - # Initialize output tensor - output = torch.empty_like(q) - - # Set grid dimensions for the kernel - grid = (B, H, D // BLOCK_SIZE) - - # Calculate strides for tensors - qkv_b_stride = q.stride(0) - qkv_h_stride = q.stride(1) - - cache_b_stride = kv_caches.stride(0) - cache_h_stride = kv_caches.stride(1) - cache_d0_stride = kv_caches.stride(2) - cache_d1_stride = kv_caches.stride(3) - - # Launch the kernel - _linear_attn_decode_kernel[grid]( - q, - k, - v, - kv_caches, - slope_rate, - slot_idx, - output, - D, - qkv_b_stride, - qkv_h_stride, - cache_b_stride, - cache_h_stride, - cache_d0_stride, - cache_d1_stride, - pad_slot_id=PAD_SLOT_ID, - BLOCK_SIZE=BLOCK_SIZE, - ) - - # Reshape output and return - output = rearrange(output, "b h n d -> b n (h d)") - return output.squeeze(1).contiguous() diff --git a/vllm/model_executor/layers/mamba/short_conv.py b/vllm/model_executor/layers/mamba/short_conv.py deleted file mode 100644 index 7a695dfd31..0000000000 --- a/vllm/model_executor/layers/mamba/short_conv.py +++ /dev/null @@ -1,382 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - - -import torch - -from vllm.compilation.breakable_cudagraph import eager_break_during_capture -from vllm.config import CacheConfig, ModelConfig, get_current_vllm_config -from vllm.distributed import get_tensor_model_parallel_world_size -from vllm.forward_context import ForwardContext, get_forward_context -from vllm.model_executor.custom_op import PluggableLayer -from vllm.model_executor.layers.linear import ( - ColumnParallelLinear, - MergedColumnParallelLinear, - RowParallelLinear, -) -from vllm.model_executor.layers.mamba.abstract import MambaBase -from vllm.model_executor.layers.mamba.mamba_utils import ( - MambaStateDtypeCalculator, - MambaStateShapeCalculator, - is_conv_state_dim_first, -) -from vllm.model_executor.layers.mamba.ops.causal_conv1d import ( - causal_conv1d_fn, - causal_conv1d_update, -) -from vllm.model_executor.layers.quantization import QuantizationConfig -from vllm.platforms import current_platform -from vllm.utils.torch_utils import direct_register_custom_op -from vllm.v1.attention.backend import AttentionMetadata -from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum -from vllm.v1.attention.backends.short_conv_attn import ShortConvAttentionMetadata - - -# --8<-- [start:short_conv] -@PluggableLayer.register("short_conv") -class ShortConv(MambaBase, PluggableLayer): - # --8<-- [end:short_conv] - - def __init__( - self, - config, - dim: int, - layer_idx: int, - model_config: ModelConfig | None = None, - cache_config: CacheConfig | None = None, - quant_config: QuantizationConfig | None = None, - prefix: str = "", - ): - super().__init__() - self.config = config - self.layer_idx = layer_idx - self.conv_dim = dim - self.L_cache = config.conv_L_cache - self.bias = config.conv_bias - - self.conv = ColumnParallelLinear( - input_size=self.L_cache, - output_size=dim, - bias=self.bias, - prefix=f"{prefix}.conv1d", - ) - # unsqueeze to fit conv1d weights shape into the linear weights shape. - # Can't do this in `weight_loader` since it already exists in - # `ColumnParallelLinear` and `set_weight_attrs` - # doesn't allow to override it - self.conv.weight.data = self.conv.weight.data.unsqueeze(1) - - self.in_proj = MergedColumnParallelLinear( - input_size=dim, - output_sizes=[dim] * 3, - bias=self.bias, - quant_config=quant_config, - prefix=f"{prefix}.in_proj", - ) - self.out_proj = RowParallelLinear( - input_size=dim, - output_size=dim, - bias=self.bias, - quant_config=quant_config, - prefix=f"{prefix}.out_proj", - ) - - vllm_config = get_current_vllm_config() - self.num_spec = vllm_config.num_speculative_tokens - compilation_config = vllm_config.compilation_config - if prefix in compilation_config.static_forward_context: - raise ValueError(f"Duplicate layer name: {prefix}") - compilation_config.static_forward_context[prefix] = self - self.kv_cache = (torch.tensor([]),) - - self.model_config = model_config - self.cache_config = cache_config - self.prefix = prefix - - def forward_native( - self, - hidden_states: torch.Tensor, - output: torch.Tensor, - ): - # Reference torch causal conv1d; runs on all CPU platforms. AMX kernels - # for causal conv can be plugged in here later. - from vllm.model_executor.layers.mamba.ops.cpu.causal_conv1d import ( - causal_conv1d_fn_cpu as causal_conv1d_torch, - ) - from vllm.model_executor.layers.mamba.ops.cpu.causal_conv1d import ( - causal_conv1d_update_cpu, - causal_conv1d_update_torch, - ) - from vllm.platforms import CpuArchEnum, current_platform - - forward_context = get_forward_context() - attn_metadata_raw = forward_context.attn_metadata - attn_metadata: AttentionMetadata | None = None - if attn_metadata_raw is not None: - assert isinstance(attn_metadata_raw, dict) - attn_metadata = attn_metadata_raw.get(self.prefix) - assert attn_metadata is None or isinstance( - attn_metadata, ShortConvAttentionMetadata - ) - - BCx, _ = self.in_proj(hidden_states) - B, C, x = BCx.chunk(3, dim=-1) - - # (dim, kernel_size) — same reshape as forward_cuda - conv_weights = self.conv.weight.view( - self.conv.weight.size(0), self.conv.weight.size(2) - ) - - if attn_metadata is None: - # Profile run — output value doesn't matter - Bx = (B * x).contiguous() - output_tensor, _ = self.out_proj(C * Bx) - output[: hidden_states.shape[0]] = output_tensor - return - - conv_state = ( - self.kv_cache[0] - if is_conv_state_dim_first() - else self.kv_cache[0].transpose(-1, -2) - ) # (num_blocks, dim, state_len) - - num_prefills = attn_metadata.num_prefills - num_decodes = attn_metadata.num_decode_tokens - num_prefill_tokens = attn_metadata.num_prefill_tokens - has_prefill = num_prefills > 0 - has_decode = num_decodes > 0 - num_actual_tokens = num_decodes + num_prefill_tokens - - B_d, B_p = torch.split( - B[:num_actual_tokens], [num_decodes, num_prefill_tokens], dim=0 - ) - C_d, C_p = torch.split( - C[:num_actual_tokens], [num_decodes, num_prefill_tokens], dim=0 - ) - x_d, x_p = torch.split( - x[:num_actual_tokens], [num_decodes, num_prefill_tokens], dim=0 - ) - - conv_output_list = [] - - if has_prefill: - assert attn_metadata.state_indices_tensor_p is not None - Bx_p = (B_p * x_p).transpose(0, 1) # (dim, num_prefill_tokens) - out_p = causal_conv1d_torch( - Bx_p, - conv_weights, - self.conv.bias, - conv_state, - attn_metadata.query_start_loc_p, - attn_metadata.state_indices_tensor_p.flatten(), - attn_metadata.has_initial_states_p, - activation=None, - ).transpose(0, 1)[:num_prefill_tokens] # (num_prefill_tokens, dim) - conv_output_list.append(C_p * out_p) - - if has_decode: - assert attn_metadata.state_indices_tensor_d is not None - state_indices_d = attn_metadata.state_indices_tensor_d.flatten() - Bx_d = B_d * x_d # (num_decodes, dim) - if current_platform.get_cpu_architecture() == CpuArchEnum.ARM: - conv_state_view = conv_state[state_indices_d].contiguous() - out_d = causal_conv1d_update_torch( - Bx_d.unsqueeze(-1), - conv_state_view, - conv_weights, - self.conv.bias, - activation=None, - ).squeeze(-1) - conv_state[state_indices_d] = conv_state_view - else: - out_d = causal_conv1d_update_cpu( - Bx_d, - conv_state, - conv_weights, - self.conv.bias, - activation=None, - conv_state_indices=state_indices_d, - ) - conv_output_list.insert(0, C_d * out_d) - - hidden_states_out = torch.vstack(conv_output_list) - output[:num_actual_tokens], _ = self.out_proj(hidden_states_out) - - def forward( - self, - hidden_states: torch.Tensor, - output: torch.Tensor, - ): - torch.ops.vllm.short_conv( - hidden_states, - output, - self.prefix, - ) - - def forward_cuda( - self, - hidden_states: torch.Tensor, - output: torch.Tensor, - ): - forward_context = get_forward_context() - # ShortConvAttentionMetadata contains metadata necessary for the - # short_conv triton kernels to operate in continuous batching and in - # chunked prefill modes; they are computed at top-level model forward - # since they stay the same and reused for all mamba layers in the same - # iteration. - attn_metadata_raw = forward_context.attn_metadata - attn_metadata: AttentionMetadata | None = None - if attn_metadata_raw is not None: - assert isinstance(attn_metadata_raw, dict) - attn_metadata = attn_metadata_raw.get(self.prefix) - if attn_metadata is not None: - assert isinstance(attn_metadata, ShortConvAttentionMetadata) - conv_state = ( - self.kv_cache[0] - if is_conv_state_dim_first() - else self.kv_cache[0].transpose(-1, -2) - ) - state_indices_tensor_p = attn_metadata.state_indices_tensor_p - state_indices_tensor_d = attn_metadata.state_indices_tensor_d - has_initial_states_p = attn_metadata.has_initial_states_p - query_start_loc_p = attn_metadata.query_start_loc_p - num_accepted_tokens = attn_metadata.num_accepted_tokens - query_start_loc_d = attn_metadata.query_start_loc_d - - BCx, _ = self.in_proj(hidden_states) - - B, C, x = BCx.chunk(3, dim=-1) - - conv_weights = self.conv.weight.view( - self.conv.weight.size(0), self.conv.weight.size(2) - ) - - if attn_metadata is None: - # V1 profile run - Bx = (B * x).contiguous() - hidden_states = C * Bx - contextualized_states, _ = self.out_proj(hidden_states) - return contextualized_states - - num_prefills = attn_metadata.num_prefills # request count - num_decodes = attn_metadata.num_decode_tokens # token count (=request) - num_prefill_tokens = attn_metadata.num_prefill_tokens # token count - has_prefill = num_prefills > 0 - has_decode = num_decodes > 0 - num_actual_tokens = num_decodes + num_prefill_tokens - - # NOTE: V1 puts decode before prefill - # Separate prefill and decode by splitting varlen input - # Split along token dimension - B_d, B_p = torch.split( - B[:num_actual_tokens], - [num_decodes, num_prefill_tokens], - dim=0, - ) - C_d, C_p = torch.split( - C[:num_actual_tokens], - [num_decodes, num_prefill_tokens], - dim=0, - ) - x_d, x_p = torch.split( - x[:num_actual_tokens], - [num_decodes, num_prefill_tokens], - dim=0, - ) - conv_output_list = [] - - if has_prefill: - Bx_p = (B_p * x_p).transpose(0, 1) - Bx = causal_conv1d_fn( - Bx_p, - conv_weights, - self.conv.bias, - activation=None, - conv_states=conv_state, - has_initial_state=has_initial_states_p, - cache_indices=state_indices_tensor_p, - metadata=attn_metadata, - query_start_loc=query_start_loc_p, - ).transpose(0, 1)[:num_prefill_tokens] - - y = C_p * Bx - conv_output_list.append(y) - - if has_decode: - Bx_d = (B_d * x_d).contiguous() - if num_accepted_tokens is not None: - # Speculative decode: the verify step feeds >1 query token per - # decode request, so use the spec-aware conv update path - # (mirrors mamba_mixer2). state_indices_tensor_d is - # (num_decodes, 1 + num_spec_tokens) here. - assert state_indices_tensor_d is not None - Bx = causal_conv1d_update( - Bx_d, - conv_state, - conv_weights, - self.conv.bias, - activation=None, - conv_state_indices=state_indices_tensor_d, - num_accepted_tokens=num_accepted_tokens, - query_start_loc=query_start_loc_d, - max_query_len=state_indices_tensor_d.size(-1), - ) - else: - # Non-spec decode - Bx = causal_conv1d_update( - Bx_d, - conv_state, - conv_weights, - self.conv.bias, - activation=None, - conv_state_indices=state_indices_tensor_d, - ) - y = C_d * Bx - conv_output_list.insert(0, y) - - # Merge prefill and decode outputs before passing to gated MLP - hidden_states = torch.vstack(conv_output_list) - - # Final linear projection - output[:num_actual_tokens], _ = self.out_proj(hidden_states) - - def get_state_dtype(self) -> tuple[torch.dtype, ...]: - assert self.model_config is not None - assert self.cache_config is not None - return MambaStateDtypeCalculator.short_conv_state_dtype( - self.model_config.dtype, - self.cache_config.mamba_cache_dtype, - ) - - def get_state_shape(self) -> tuple[tuple[int, ...]]: - return MambaStateShapeCalculator.short_conv_state_shape( - tp_world_size=get_tensor_model_parallel_world_size(), - intermediate_size=self.conv_dim, - conv_kernel=self.L_cache, - num_spec=self.num_spec, - ) - - @property - def mamba_type(self) -> MambaAttentionBackendEnum: - return MambaAttentionBackendEnum.SHORT_CONV - - -@eager_break_during_capture -def short_conv( - hidden_states: torch.Tensor, - output: torch.Tensor, - layer_name: str, -) -> None: - forward_context: ForwardContext = get_forward_context() - self = forward_context.no_compile_layers[layer_name] - if not current_platform.is_cpu(): - self.forward_cuda(hidden_states=hidden_states, output=output) - else: - self.forward_native(hidden_states=hidden_states, output=output) - - -direct_register_custom_op( - op_name="short_conv", - op_func=short_conv, - mutates_args=["output"], -) diff --git a/vllm/model_executor/layers/minimax_rms_norm/__init__.py b/vllm/model_executor/layers/minimax_rms_norm/__init__.py deleted file mode 100644 index 458a6905c9..0000000000 --- a/vllm/model_executor/layers/minimax_rms_norm/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from vllm.model_executor.layers.minimax_rms_norm.rms_norm_tp import ( - MiniMaxText01RMSNormTP, -) - -__all__ = [ - "MiniMaxText01RMSNormTP", -] diff --git a/vllm/model_executor/layers/minimax_rms_norm/lamport_workspace.py b/vllm/model_executor/layers/minimax_rms_norm/lamport_workspace.py deleted file mode 100644 index eac2a64a04..0000000000 --- a/vllm/model_executor/layers/minimax_rms_norm/lamport_workspace.py +++ /dev/null @@ -1,316 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - - -import array -import contextlib -import ctypes -import struct -import sys -import threading - -import torch - -try: - from cuda.bindings import runtime as cudart -except ImportError: - from cuda import cudart - -_ALIGN = 1 << 21 # 2 MiB — CUDA IPC allocation alignment -_IPC_HANDLE_SIZE = 64 # sizeof(cudaIpcMemHandle_t) - - -# --------------------------------------------------------------------------- -# CUDA helpers -# --------------------------------------------------------------------------- - - -def _check(error): - """Raise on CUDA runtime error.""" - success = getattr(cudart.cudaError_t, "cudaSuccess", None) or cudart.cudaError_t(0) - if error != success: - raise RuntimeError(f"CUDA runtime error: {error}") - - -# cuda-bindings >= 13.4 no longer exposes cudaIpcMemHandle_t.reserved, so -# copy the raw handle bytes through getPtr() instead. This is safe because the -# CUDA ABI fixes the handle as an opaque 64-byte struct (char reserved[64]). -def _ipc_handle_to_bytes(handle) -> bytes: - return ctypes.string_at(handle.getPtr(), _IPC_HANDLE_SIZE) - - -def _ipc_handle_from_bytes(raw: bytes): - assert len(raw) == _IPC_HANDLE_SIZE, len(raw) - handle = cudart.cudaIpcMemHandle_t() - ctypes.memmove(handle.getPtr(), raw, _IPC_HANDLE_SIZE) - return handle - - -def _cuda_malloc(size: int): - aligned = ((size + _ALIGN - 1) >> 21) << 21 - err, ptr = cudart.cudaMalloc(aligned) - _check(err) - return ptr, aligned - - -def _cuda_free(ptr: int): - if ptr: - _check(cudart.cudaFree(ptr)[0]) - - -def _cuda_memset_zero(ptr: int, size: int): - _check(cudart.cudaMemset(ptr, 0, size)[0]) - - -def _cuda_memcpy_d2d(dst: int, src: int, size: int): - _check( - cudart.cudaMemcpy( - dst, src, size, cudart.cudaMemcpyKind.cudaMemcpyDeviceToDevice - )[0] - ) - - -# --------------------------------------------------------------------------- -# IPC buffer -# --------------------------------------------------------------------------- - - -class IpcBuffer: - """Allocates CUDA device memory and exchanges IPC handles with all ranks - so that every rank holds a valid device pointer to every other rank's buffer. - """ - - def __init__(self, rank: int, world_size: int, size: int, process_group=None): - self.rank = rank - self.world_size = world_size - self.peer_ptrs: list[int] = [0] * world_size - self.local_ptr: int = 0 - self._alive = False - - if size <= 0: - return - - self.local_ptr, _ = _cuda_malloc(size) - _cuda_memset_zero(self.local_ptr, size) - self._alive = True - - # --- exchange IPC handles via torch.distributed --- - err, local_handle = cudart.cudaIpcGetMemHandle(self.local_ptr) - _check(err) - - all_handles: list[bytes | None] = [None] * world_size - torch.distributed.all_gather_object( - all_handles, _ipc_handle_to_bytes(local_handle), group=process_group - ) - - for r in range(world_size): - if r == rank: - self.peer_ptrs[r] = self.local_ptr - else: - raw = all_handles[r] - assert raw is not None - handle = _ipc_handle_from_bytes(raw) - err, ptr = cudart.cudaIpcOpenMemHandle( - handle, cudart.cudaIpcMemLazyEnablePeerAccess - ) - _check(err) - self.peer_ptrs[r] = ptr - - def serialize(self) -> list[int]: - """Return peer pointers as a list of int64 values (one per rank).""" - raw = b"" - for ptr in self.peer_ptrs: - raw += struct.pack("P", ptr) - return array.array("Q", raw).tolist() - - def cleanup(self): - if not self._alive: - return - self._alive = False - for r in range(self.world_size): - if self.peer_ptrs[r] == 0: - continue - if r == self.rank: - _cuda_free(self.peer_ptrs[r]) - else: - with contextlib.suppress(RuntimeError): - _check(cudart.cudaIpcCloseMemHandle(self.peer_ptrs[r])[0]) - self.peer_ptrs[r] = 0 - self.local_ptr = 0 - - def __del__(self): - if not sys.is_finalizing(): - self.cleanup() - - -# --------------------------------------------------------------------------- -# Lamport negative-zero initialization -# --------------------------------------------------------------------------- - - -def _lamport_fill_neg_zero(device_ptr: int, size_bytes: int): - """Fill device memory with IEEE-754 negative zero (-0.0f = 0x80000000). - This is the "slot empty" sentinel for the Lamport protocol: the kernel - spin-waits until a value is *not* negative zero. - """ - if size_bytes == 0 or device_ptr == 0: - return - n_floats = size_bytes // 4 - # torch preserves -0.0 in IEEE-754 - fill = torch.full((n_floats,), -0.0, dtype=torch.float32, device="cuda") - _cuda_memcpy_d2d(device_ptr, fill.data_ptr(), size_bytes) - del fill - - -# --------------------------------------------------------------------------- -# LamportWorkspace — the main class -# --------------------------------------------------------------------------- - - -class LamportWorkspace: - """Self-contained workspace for Lamport-based cross-GPU AllReduce. - - Parameters - ---------- - rank : int - Local rank (0-based). - world_size : int - Total number of ranks in the TP group. - comm_size : int - Size in bytes of *one* Lamport buffer slot. The total IPC allocation - per rank is ``3 * comm_size`` (triple-buffering). Must be large enough - to hold the per-slot data written by the kernel. Use - ``compute_comm_size_for_minimax()`` for a safe default. - process_group : optional - ``torch.distributed`` process group for IPC handle exchange. - ``None`` uses the default group. - - """ - - def __init__(self, rank: int, world_size: int, comm_size: int, process_group=None): - assert world_size >= 2, "Lamport workspace requires at least 2 ranks" - assert comm_size > 0, "comm_size must be positive" - - self.rank = rank - self.world_size = world_size - self.comm_size = comm_size - - # 1) Lamport triple-buffer (the only IPC memory the kernel reads/writes) - lamport_total = 3 * comm_size - self._lamport = IpcBuffer(rank, world_size, lamport_total, process_group) - _lamport_fill_neg_zero(self._lamport.local_ptr, lamport_total) - - # 2) flag_buffer on device: int32[3] = {counter, unused, lamport_flag} - # counter — used for block-level sync inside the kernel - # unused — reserved (index 1) - # lamport_flag — triple-buffer rotation index (0 → 1 → 2 → 0 …) - self._flag_buf = torch.zeros(3, dtype=torch.int32, device="cuda") - - # 3) layout_buffer on device: int64[2] = {clear_size, comm_size} - # clear_size — bytes to clear from *previous* slot (set by kernel) - # comm_size — size of one triple-buffer slot - self._layout_buf = torch.tensor( - [0, comm_size], dtype=torch.int64, device="cuda" - ) - - # 4) Assemble device-side void* pointer array - N = world_size - ptrs: list[int] = [] - ptrs += [0] * N # [0 .. N-1] ipc_buffers (placeholder) - ptrs += [0] * N # [N .. 2N-1] ipc_barriers (placeholder) - ptrs += self._lamport.serialize() # [2N .. 3N-1] lamport peer ptrs - ptrs.append(self._flag_buf.data_ptr()) # [3N] flag_buffer - ptrs.append(self._layout_buf.data_ptr()) # [3N+1] layout_buffer - - self._workspace = torch.tensor(ptrs, dtype=torch.int64, device="cuda") - - @property - def workspace(self) -> torch.Tensor: - """Device tensor (int64) that can be passed to the kernel - as ``void** workspace``.""" - return self._workspace - - # ------------------------------------------------------------------ - # Helpers - # ------------------------------------------------------------------ - - @staticmethod - def compute_comm_size_for_minimax( - max_tokens: int, - world_size: int, - fused_qk: bool = True, - ) -> int: - """Return a safe ``comm_size`` (in bytes) for MiniMaxReduceRMSKernel. - - The kernel stores per-token variance scalars in the Lamport buffer: - - single-matrix path: ``world_size × max_tokens × 4`` bytes per slot - - fused Q+K path: ``world_size × 2 × ceil(max_tokens/4) × 16`` bytes per slot - - The returned value is rounded up to 2 MiB alignment. - """ - if fused_qk: - groups = (max_tokens + 3) // 4 - slot_bytes = world_size * 2 * groups * 16 # 16 = sizeof(float4) - else: - slot_bytes = world_size * max_tokens * 4 # 4 = sizeof(float) - return ((slot_bytes + _ALIGN - 1) >> 21) << 21 - - def cleanup(self): - if hasattr(self, "_lamport"): - self._lamport.cleanup() - - def __del__(self): - if not sys.is_finalizing(): - self.cleanup() - - def __repr__(self): - return ( - f"LamportWorkspace(rank={self.rank}, world_size={self.world_size}, " - f"comm_size={self.comm_size})" - ) - - -# --------------------------------------------------------------------------- -# Cached convenience function (mirrors TRT-LLM's get_allreduce_workspace) -# --------------------------------------------------------------------------- - -_cache_lock = threading.Lock() -_workspace_cache: dict = {} - - -def get_allreduce_workspace( - rank: int, - world_size: int, - comm_size: int | None = None, - max_tokens: int = 16384, - process_group=None, -) -> torch.Tensor: - """Return a cached workspace tensor for the given (rank, world_size) pair. - - On first call the workspace is allocated and IPC handles are exchanged; - subsequent calls with the same arguments return the cached tensor. - - Parameters - ---------- - rank, world_size : int - TP rank and TP size. - comm_size : int, optional - Explicit slot size in bytes. If ``None``, computed automatically - from ``max_tokens`` and ``world_size`` (fused Q+K path). - max_tokens : int - Maximum number of tokens per batch (used when ``comm_size is None``). - process_group : optional - ``torch.distributed`` process group. - - """ - if comm_size is None: - comm_size = LamportWorkspace.compute_comm_size_for_minimax( - max_tokens, world_size, fused_qk=True - ) - pg_id = id(process_group) if process_group is not None else 0 - key = (rank, world_size, comm_size, pg_id) - with _cache_lock: - if key not in _workspace_cache: - ws = LamportWorkspace(rank, world_size, comm_size, process_group) - _workspace_cache[key] = ws - return _workspace_cache[key].workspace diff --git a/vllm/model_executor/layers/minimax_rms_norm/rms_norm_tp.py b/vllm/model_executor/layers/minimax_rms_norm/rms_norm_tp.py deleted file mode 100644 index 5094662f74..0000000000 --- a/vllm/model_executor/layers/minimax_rms_norm/rms_norm_tp.py +++ /dev/null @@ -1,421 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from functools import partial - -import torch -from torch import nn - -from vllm._aiter_ops import rocm_aiter_ops -from vllm.distributed.communication_op import tensor_model_parallel_all_reduce -from vllm.distributed.parallel_state import ( - get_tensor_model_parallel_rank, - get_tensor_model_parallel_world_size, - get_tp_group, -) -from vllm.logger import init_logger -from vllm.model_executor.custom_op import CustomOp -from vllm.triton_utils import HAS_TRITON, tl, triton -from vllm.utils.torch_utils import direct_register_custom_op - -logger = init_logger(__name__) - -# Max number of tokens supported by the Lamport fused allreduce+RMSNorm kernel. -# Larger batches fall back to the eager allreduce + RMSNorm path. -MINIMAX_QK_NORM_MAX_TOKEN_NUM = 2048 - -_MINIMAX_FUSED_AR_RMS_QK = getattr(torch.ops._C, "minimax_allreduce_rms_qk", None) - - -def _all_reduce_variance(var: torch.Tensor) -> torch.Tensor: - """All-reduce a per-token variance tensor across the TP group. - - Variance is accumulated in fp32 for numerical stability. The FlashInfer - fused all-reduce caches a single global workspace keyed to the model's - 16-bit activation dtype (``use_fp32_lamport=False``); routing an fp32 - reduction through it would read against a mismatched workspace and corrupt - the result. FlashInfer's fast-path only triggers for 2D inputs, so reducing - a flattened (1D) view keeps these fp32 reductions on custom all-reduce / - pynccl, both of which handle fp32 correctly. - """ - return tensor_model_parallel_all_reduce(var.flatten()).view_as(var) - - -@triton.jit -def _minimax_qk_var_kernel( - qkv_ptr, # [num_tokens, hidden], 16-bit activations - var_ptr, # [num_tokens, 2], fp32 - row_stride, # element stride between tokens in qkv - q_size: tl.constexpr, # constant per deployment -> loops unroll, mask elides - kv_size: tl.constexpr, - BLOCK: tl.constexpr, -): - """TP-pre stage: per-token mean-of-squares for the q and k segments. - - Accumulates in fp32 while reading the 16-bit qkv in place, so no fp32 - copy of q/k is materialized. ``var[:, 0]`` is the q variance and - ``var[:, 1]`` the k variance; both are the local-shard means, ready for - the all-reduce that follows. - """ - token = tl.program_id(0) - base = qkv_ptr + token * row_stride - - q_acc = 0.0 - for off in range(0, q_size, BLOCK): - idx = off + tl.arange(0, BLOCK) - mask = idx < q_size - x = tl.load(base + idx, mask=mask, other=0.0).to(tl.float32) - q_acc += tl.sum(x * x, axis=0) - - k_acc = 0.0 - for off in range(0, kv_size, BLOCK): - idx = off + tl.arange(0, BLOCK) - mask = idx < kv_size - x = tl.load(base + q_size + idx, mask=mask, other=0.0).to(tl.float32) - k_acc += tl.sum(x * x, axis=0) - - tl.store(var_ptr + token * 2 + 0, q_acc / q_size) - tl.store(var_ptr + token * 2 + 1, k_acc / kv_size) - - -@triton.jit -def _minimax_rms_apply_kernel( - qkv_ptr, # [num_tokens, hidden] - var_ptr, # [num_tokens, 2], fp32, all-reduced sum of per-shard means - q_w_ptr, # [q_size], q per-channel weight - k_w_ptr, # [kv_size], k per-channel weight - q_out_ptr, # [num_tokens, q_size], contiguous - k_out_ptr, # [num_tokens, kv_size], contiguous - row_stride, # element stride between tokens in qkv - q_size: tl.constexpr, # constant per deployment -> loops unroll, mask elides - kv_size: tl.constexpr, - tp_world: tl.constexpr, # folds the post-all-reduce /tp_world into rsqrt - eps: tl.constexpr, - BLOCK: tl.constexpr, -): - """TP-post stage: ``x * rsqrt(var / tp_world + eps) * weight``. - - A single program normalizes both the q and k segments of one token, so q - and k share one launch instead of two. The all-reduce yields the sum of - per-shard means, so the ``/ tp_world`` that recovers the global - mean-of-squares is folded into the ``rsqrt`` here rather than run as a - separate elementwise pass over the ``[num_tokens, 2]`` variance tensor. - """ - token = tl.program_id(0) - base = qkv_ptr + token * row_stride - - q_inv = tl.rsqrt(tl.load(var_ptr + token * 2 + 0) / tp_world + eps) - q_out_row = q_out_ptr + token * q_size - for off in range(0, q_size, BLOCK): - idx = off + tl.arange(0, BLOCK) - mask = idx < q_size - x = tl.load(base + idx, mask=mask, other=0.0).to(tl.float32) - w = tl.load(q_w_ptr + idx, mask=mask, other=0.0).to(tl.float32) - y = x * q_inv * w - tl.store(q_out_row + idx, y.to(q_out_ptr.dtype.element_ty), mask=mask) - - k_inv = tl.rsqrt(tl.load(var_ptr + token * 2 + 1) / tp_world + eps) - k_out_row = k_out_ptr + token * kv_size - for off in range(0, kv_size, BLOCK): - idx = off + tl.arange(0, BLOCK) - mask = idx < kv_size - x = tl.load(base + q_size + idx, mask=mask, other=0.0).to(tl.float32) - w = tl.load(k_w_ptr + idx, mask=mask, other=0.0).to(tl.float32) - y = x * k_inv * w - tl.store(k_out_row + idx, y.to(k_out_ptr.dtype.element_ty), mask=mask) - - -def _minimax_qk_norm_tp_eager( - qkv: torch.Tensor, - q_weight: torch.Tensor, - k_weight: torch.Tensor, - q_size: int, - kv_size: int, - tp_world: int, - eps: float, -) -> tuple[torch.Tensor, torch.Tensor]: - """Pure-torch reference path used when Triton is unavailable.""" - q, k, _ = qkv.split([q_size, kv_size, kv_size], dim=-1) - orig_dtype = q.dtype - q = q.to(torch.float32) - k = k.to(torch.float32) - q_var = q.pow(2).mean(dim=-1, keepdim=True) - k_var = k.pow(2).mean(dim=-1, keepdim=True) - - qk_var = torch.cat([q_var, k_var], dim=-1) - qk_var = _all_reduce_variance(qk_var) / tp_world - q_var, k_var = qk_var.chunk(2, dim=-1) - q = q * torch.rsqrt(q_var + eps) * q_weight - k = k * torch.rsqrt(k_var + eps) * k_weight - return q.to(orig_dtype), k.to(orig_dtype) - - -def _minimax_qk_norm_tp_fallback( - qkv: torch.Tensor, - q_weight: torch.Tensor, - k_weight: torch.Tensor, - q_size: int, - kv_size: int, - tp_rank: int, - tp_world: int, - eps: float, -) -> tuple[torch.Tensor, torch.Tensor]: - """All-reduce + QK RMSNorm without the Lamport fused kernel. - - The all-reduce is a TP communication barrier and cannot live inside a - single kernel, so the eager-torch path is split into two Triton kernels - around it: a variance reduction before the all-reduce and a normalize - after. Compared to the ``torch.compile`` path this avoids materializing - fp32 copies of q/k and the ``cat``/``chunk`` temporaries. - """ - if not HAS_TRITON: - return _minimax_qk_norm_tp_eager( - qkv, q_weight, k_weight, q_size, kv_size, tp_world, eps - ) - - num_tokens = qkv.shape[0] - row_stride = qkv.stride(0) - BLOCK = 1024 - grid = (num_tokens,) - - qk_var = torch.empty(num_tokens, 2, dtype=torch.float32, device=qkv.device) - _minimax_qk_var_kernel[grid]( - qkv, qk_var, row_stride, q_size=q_size, kv_size=kv_size, BLOCK=BLOCK - ) - - # All-reduce sums the per-shard means; the /tp_world that turns this back - # into the global mean is folded into the apply kernel's rsqrt below. - qk_var = _all_reduce_variance(qk_var) - - q_out = torch.empty(num_tokens, q_size, dtype=qkv.dtype, device=qkv.device) - k_out = torch.empty(num_tokens, kv_size, dtype=qkv.dtype, device=qkv.device) - _minimax_rms_apply_kernel[grid]( - qkv, - qk_var, - q_weight, - k_weight, - q_out, - k_out, - row_stride, - q_size=q_size, - kv_size=kv_size, - tp_world=tp_world, - eps=eps, - BLOCK=BLOCK, - ) - return q_out, k_out - - -def _minimax_qk_norm_fusion( - qkv: torch.Tensor, - q_weight: torch.Tensor, - k_weight: torch.Tensor, - q_size: int, - kv_size: int, - tp_rank: int, - tp_world: int, - eps: float, - workspace: torch.Tensor | None, -) -> tuple[torch.Tensor, torch.Tensor]: - assert qkv.ndim == 2 - num_tokens = qkv.shape[0] - if ( - workspace is not None - and tp_world > 1 - and num_tokens <= MINIMAX_QK_NORM_MAX_TOKEN_NUM - and _MINIMAX_FUSED_AR_RMS_QK is not None - ): - return _MINIMAX_FUSED_AR_RMS_QK( - qkv, - q_weight, - k_weight, - workspace, - q_size, - kv_size, - tp_rank, - tp_world, - eps, - ) - # ROCm AITER fused QK-norm + AllReduce; skips unaligned shapes. - if tp_world > 1 and num_tokens <= MINIMAX_QK_NORM_MAX_TOKEN_NUM: - pack_size = 16 // qkv.element_size() - warp_work_size = 32 * pack_size - if q_size % warp_work_size == 0 and kv_size % warp_work_size == 0: - aiter_ar = rocm_aiter_ops.get_aiter_allreduce() - if aiter_ar is not None: - q_out, k_out, _ = aiter_ar.aiter_ca.custom_fused_qknorm_ar( - qkv, q_weight, k_weight, eps - ) - return q_out, k_out - return _minimax_qk_norm_tp_fallback( - qkv, q_weight, k_weight, q_size, kv_size, tp_rank, tp_world, eps - ) - - -def _minimax_qk_norm_fusion_fake( - qkv: torch.Tensor, - q_weight: torch.Tensor, - k_weight: torch.Tensor, - q_size: int, - kv_size: int, - tp_rank: int, - tp_world: int, - eps: float, - workspace: torch.Tensor | None, -) -> tuple[torch.Tensor, torch.Tensor]: - assert qkv.ndim == 2 - num_tokens = qkv.shape[0] - return ( - torch.empty([num_tokens, q_size], dtype=qkv.dtype, device=qkv.device), - torch.empty([num_tokens, kv_size], dtype=qkv.dtype, device=qkv.device), - ) - - -direct_register_custom_op( - op_name="minimax_qk_norm_fusion", - op_func=_minimax_qk_norm_fusion, - fake_impl=_minimax_qk_norm_fusion_fake, - mutates_args=[], -) - - -@CustomOp.register("minimax_text01_rmsnorm_tp") -class MiniMaxText01RMSNormTP(CustomOp): - def __init__( - self, - hidden_size: int, - eps: float = 1e-6, - *, - weight_shard_world_size: int | None = None, - weight_shard_rank: int | None = None, - ) -> None: - super().__init__() - self.tp_world = get_tensor_model_parallel_world_size() - self.tp_rank = get_tensor_model_parallel_rank() - self.weight_shard_world = weight_shard_world_size or self.tp_world - self.weight_shard_rank = ( - self.tp_rank if weight_shard_rank is None else weight_shard_rank - ) - - self.weight = nn.Parameter(torch.ones(hidden_size // self.weight_shard_world)) - self.weight.weight_loader = partial( - self.weight_loader, - shard_world_size=self.weight_shard_world, - shard_rank=self.weight_shard_rank, - ) - self.variance_epsilon = eps - - self.workspace = None - if _MINIMAX_FUSED_AR_RMS_QK is not None and self.tp_world > 1: - from .lamport_workspace import ( - get_allreduce_workspace, - ) - - # The Lamport workspace exchanges CUDA IPC handles and enables peer - # access between GPUs. This requires P2P (IPC peer access) to be - # available; on topologies where it is not (e.g. consumer PCIe cards - # with P2P disabled in the driver), allocation raises. Fall back to - # the eager allreduce + RMSNorm path instead of failing model load. - try: - self.workspace = get_allreduce_workspace( - rank=self.tp_rank, - world_size=self.tp_world, - max_tokens=MINIMAX_QK_NORM_MAX_TOKEN_NUM, - process_group=get_tp_group().cpu_group, - ) - except Exception as e: - logger.warning_once( - "Failed to initialize MiniMax fused allreduce+RMSNorm " - "Lamport workspace: %s. This is expected on GPUs without " - "P2P (IPC peer access) support. Falling back to the eager " - "allreduce + RMSNorm path.", - e, - ) - self.workspace = None - - @staticmethod - def weight_loader( - param: nn.Parameter, - loaded_weight: torch.Tensor, - shard_world_size: int | None = None, - shard_rank: int | None = None, - ) -> None: - if shard_world_size is None: - shard_world_size = get_tensor_model_parallel_world_size() - if shard_rank is None: - shard_rank = get_tensor_model_parallel_rank() - - shard_size = loaded_weight.shape[0] // shard_world_size - shard = slice(shard_rank * shard_size, (shard_rank + 1) * shard_size) - param.data.copy_(loaded_weight[shard]) - - def _forward( - self, - x: torch.Tensor, - ) -> torch.Tensor: - orig_dtype = x.dtype - x = x.to(torch.float32) - variance = x.pow(2).mean(dim=-1, keepdim=True, dtype=torch.float32) - if self.tp_world > 1: - variance = _all_reduce_variance(variance) / self.tp_world - x = x * torch.rsqrt(variance + self.variance_epsilon) - x = (x * self.weight).to(orig_dtype) - return x - - def forward( - self, - x: torch.Tensor, - residual: torch.Tensor | None = None, - ) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]: - assert residual is None, "RMSNorm does not support residual connection." - return self._forward(x) - - @staticmethod - def forward_qk( - q_norm: "MiniMaxText01RMSNormTP", - k_norm: "MiniMaxText01RMSNormTP", - q: torch.Tensor, - k: torch.Tensor, - ) -> tuple[torch.Tensor, torch.Tensor]: - orig_dtype = q.dtype - q = q.to(torch.float32) - k = k.to(torch.float32) - q_var = q.pow(2).mean(dim=-1, keepdim=True) - k_var = k.pow(2).mean(dim=-1, keepdim=True) - - q = q * torch.rsqrt(q_var + q_norm.variance_epsilon) * q_norm.weight - k = k * torch.rsqrt(k_var + k_norm.variance_epsilon) * k_norm.weight - q = q.to(orig_dtype) - k = k.to(orig_dtype) - return q, k - - @staticmethod - def forward_qkv( - q_norm: "MiniMaxText01RMSNormTP", - k_norm: "MiniMaxText01RMSNormTP", - qkv: torch.Tensor, - q_size: int, - kv_size: int, - ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - assert qkv.ndim == 2 - - assert q_norm.variance_epsilon == k_norm.variance_epsilon - # Case 0: tp_size=1 - if get_tensor_model_parallel_world_size() == 1: - q, k, v = qkv.split([q_size, kv_size, kv_size], dim=-1) - q, k = MiniMaxText01RMSNormTP.forward_qk(q_norm, k_norm, q, k) - return q, k, v - # Case : tp_size>1 - q, k = torch.ops.vllm.minimax_qk_norm_fusion( - qkv, - q_norm.weight, - k_norm.weight, - q_size, - kv_size, - q_norm.tp_rank, - q_norm.tp_world, - q_norm.variance_epsilon, - q_norm.workspace, - ) - _, _, v = qkv.split([q_size, kv_size, kv_size], dim=-1) - return q, k, v diff --git a/vllm/model_executor/models/registry.py b/vllm/model_executor/models/registry.py index 27542fe0ec..328513430b 100644 --- a/vllm/model_executor/models/registry.py +++ b/vllm/model_executor/models/registry.py @@ -115,77 +115,9 @@ _SPECULATIVE_DECODING_MODELS = { "Qwen3_5MoeMTP": ("qwen3_5_mtp", "Qwen3_5MoeMTP"), } -_TRANSFORMERS_SUPPORTED_MODELS = { - # Text generation models - "ArceeForCausalLM": ("transformers", "TransformersForCausalLM"), - "CwmForCausalLM": ("transformers", "TransformersForCausalLM"), - "FlexOlmoForCausalLM": ("transformers", "TransformersMoEForCausalLM"), - "GlmForCausalLM": ("transformers", "TransformersForCausalLM"), - "GPTBigCodeForCausalLM": ("transformers", "TransformersForCausalLM"), - "GPTNeoXForCausalLM": ("transformers", "TransformersForCausalLM"), - "HunYuanDenseV1ForCausalLM": ("transformers", "TransformersForCausalLM"), - "HunYuanMoEV1ForCausalLM": ("transformers", "TransformersMoEForCausalLM"), - "Jais2ForCausalLM": ("transformers", "TransformersForCausalLM"), - "Kolibri1ForCausalLM": ("transformers", "TransformersMoEForCausalLM"), - "MellumForCausalLM": ("transformers", "TransformersMoEForCausalLM"), - "NanbeigeForCausalLM": ("transformers", "TransformersForCausalLM"), - "OlmoForCausalLM": ("transformers", "TransformersForCausalLM"), - "Olmo2ForCausalLM": ("transformers", "TransformersForCausalLM"), - "Olmo3ForCausalLM": ("transformers", "TransformersForCausalLM"), - "PhiForCausalLM": ("transformers", "TransformersForCausalLM"), - "SeedOssForCausalLM": ("transformers", "TransformersForCausalLM"), - "SmolLM3ForCausalLM": ("transformers", "TransformersForCausalLM"), - "Starcoder2ForCausalLM": ("transformers", "TransformersForCausalLM"), - "VaultGemmaForCausalLM": ("transformers", "TransformersForCausalLM"), - # Multimodal models - "Emu3ForConditionalGeneration": ( - "transformers", - "TransformersMultiModalForCausalLM", - ), - "HunYuanVLForConditionalGeneration": ( - "transformers", - "TransformersMultiModalForCausalLM", - ), - "VibeVoiceAsrForConditionalGeneration": ( - "transformers", - "TransformersMultiModalForCausalLM", - ), -} +_TRANSFORMERS_SUPPORTED_MODELS = {} -_TRANSFORMERS_BACKEND_MODELS = { - # Text generation models - "TransformersForCausalLM": ("transformers", "TransformersForCausalLM"), - "TransformersMoEForCausalLM": ("transformers", "TransformersMoEForCausalLM"), - # Multimodal models - "TransformersMultiModalForCausalLM": ( - "transformers", - "TransformersMultiModalForCausalLM", - ), - "TransformersMultiModalMoEForCausalLM": ( - "transformers", - "TransformersMultiModalMoEForCausalLM", - ), - # Embedding models - "TransformersEmbeddingModel": ("transformers", "TransformersEmbeddingModel"), - "TransformersMoEEmbeddingModel": ("transformers", "TransformersMoEEmbeddingModel"), - "TransformersMultiModalEmbeddingModel": ( - "transformers", - "TransformersMultiModalEmbeddingModel", - ), - # Sequence classification models - "TransformersForSequenceClassification": ( - "transformers", - "TransformersForSequenceClassification", - ), - "TransformersMoEForSequenceClassification": ( - "transformers", - "TransformersMoEForSequenceClassification", - ), - "TransformersMultiModalForSequenceClassification": ( - "transformers", - "TransformersMultiModalForSequenceClassification", - ), -} +_TRANSFORMERS_BACKEND_MODELS = {} _VLLM_MODELS = { **_TEXT_GENERATION_MODELS, diff --git a/vllm/model_executor/models/transformers/__init__.py b/vllm/model_executor/models/transformers/__init__.py deleted file mode 100644 index 38dd71419d..0000000000 --- a/vllm/model_executor/models/transformers/__init__.py +++ /dev/null @@ -1,155 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -# Copyright 2024 The vLLM team. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Wrapper around `transformers` models.""" - -from typing import TYPE_CHECKING - -import torch.nn.functional as F -from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS - -from vllm.model_executor.models.transformers.base import VLLM_ATTN_ATTR -from vllm.model_executor.models.transformers.causal import CausalMixin -from vllm.model_executor.models.transformers.moe import MoEMixin -from vllm.model_executor.models.transformers.multimodal import ( - MultiModalDummyInputsBuilder, - MultiModalMixin, - MultiModalProcessingInfo, - MultiModalProcessor, -) -from vllm.multimodal import MULTIMODAL_REGISTRY - -if TYPE_CHECKING: - import torch - - from vllm.model_executor.layers.attention import Attention, MLAAttention - - -def check_sinks( - module: "torch.nn.Module", - self_attn: "Attention | MLAAttention", - s_aux: "torch.Tensor | None", -): - """Fail loudly if the model applies a sink the attention layer will not. - - Only the attention impl can fold a sink into the softmax denominator, so a sink - that never reached `Attention` is dropped and every softmax is subtly wrong. - """ - if s_aux is None or getattr(self_attn, "has_sink", False): - return - raise ValueError( - f"{type(module).__name__} applies attention sinks, but they were not passed " - f"to {type(self_attn).__name__}, so the output would be wrong. Either the " - "Transformers modeling backend could not find the parameter holding them, or " - "vLLM does not support sinks for this kind of attention. Please open an issue " - "at https://github.com/vllm-project/vllm/issues/new" - ) - - -def vllm_attention_forward( - # Transformers args - module: "torch.nn.Module", - query: "torch.Tensor", - key: "torch.Tensor", - value: "torch.Tensor", - attention_mask: "torch.Tensor", - **kwargs, -): - self_attn = getattr(module, VLLM_ATTN_ATTR) - check_sinks(module, self_attn, kwargs.get("s_aux")) - hidden = query.shape[-2] - head_dim_qk = query.shape[-1] - head_dim_v = value.shape[-1] - query, key, value = (x.transpose(1, 2) for x in (query, key, value)) - query, key, value = (x.reshape(hidden, -1) for x in (query, key, value)) - # Pad `value` up to the query/key head size when it is smaller (expanded - # MLA). A larger last dim just means `value` isn't split per head, e.g. - # packed grouped/multi-query projections, and needs no padding. - pad_value = head_dim_v < head_dim_qk - if pad_value: - value = F.pad(value.view(-1, head_dim_v), (0, head_dim_qk - head_dim_v)) - value = value.reshape(hidden, -1) - attn_output = self_attn.forward(query, key, value) - if pad_value: - attn_output = attn_output.view(-1, head_dim_qk)[..., :head_dim_v] - attn_output = attn_output.reshape(hidden, -1) - return attn_output, None - - -def vllm_mla_attention_forward( - # Transformers args - module: "torch.nn.Module", - query: "torch.Tensor", - kv_c_normed: "torch.Tensor", - k_pe: "torch.Tensor", - attention_mask: "torch.Tensor", - **kwargs, -): - self_attn = getattr(module, VLLM_ATTN_ATTR) - check_sinks(module, self_attn, kwargs.get("s_aux")) - # [batch=1, heads, num_tokens, qk_head_dim] -> [num_tokens, heads, qk_head_dim] - query = query.transpose(1, 2).flatten(0, 1) - num_tokens, num_heads = query.shape[:2] - # [batch=1, num_tokens, kv_lora_rank] -> [num_tokens, kv_lora_rank] - kv_c_normed = kv_c_normed.reshape(-1, kv_c_normed.shape[-1]) - # [batch=1, heads=1, num_tokens, qk_rope] -> [num_tokens, 1, qk_rope] - k_pe = k_pe.reshape(-1, 1, k_pe.shape[-1]) - attn_output = self_attn.forward( - query, - kv_c_normed, - k_pe, - output_shape=(num_tokens, num_heads * self_attn.v_head_dim), - ) - return attn_output, None - - -ALL_ATTENTION_FUNCTIONS.register("vllm", vllm_attention_forward) -ALL_ATTENTION_FUNCTIONS.register("vllm_mla", vllm_mla_attention_forward) - - -# Text only models -class TransformersForCausalLM(CausalMixin): ... - - -class TransformersMoEForCausalLM(MoEMixin, CausalMixin): ... - - -# Multimodal models -@MULTIMODAL_REGISTRY.register_processor( - MultiModalProcessor, - info=MultiModalProcessingInfo, - dummy_inputs=MultiModalDummyInputsBuilder, -) -class TransformersMultiModalForCausalLM(MultiModalMixin, CausalMixin): ... - - -@MULTIMODAL_REGISTRY.register_processor( - MultiModalProcessor, - info=MultiModalProcessingInfo, - dummy_inputs=MultiModalDummyInputsBuilder, -) -class TransformersMultiModalMoEForCausalLM(MoEMixin, MultiModalMixin, CausalMixin): ... - - -def __getattr__(name: str): - """Handle imports of non-existent classes with a helpful error message.""" - if name not in globals(): - raise AttributeError( - "The Transformers modeling backend does not currently have a class to " - f"handle the requested model type: {name}. Please open an issue at " - "https://github.com/vllm-project/vllm/issues/new" - ) - return globals()[name] diff --git a/vllm/model_executor/models/transformers/base.py b/vllm/model_executor/models/transformers/base.py deleted file mode 100644 index cbf2aba9b8..0000000000 --- a/vllm/model_executor/models/transformers/base.py +++ /dev/null @@ -1,860 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -# Copyright 2024 The vLLM team. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Transformers modeling backend base class.""" - -import os -from collections.abc import Callable, Iterable -from contextlib import contextmanager -from functools import cached_property -from itertools import chain -from operator import attrgetter -from typing import TYPE_CHECKING, Any, NamedTuple - -import regex as re -import torch -import transformers -from packaging.version import Version -from torch import nn -from transformers import AutoModel, PreTrainedConfig -from transformers.conversion_mapping import ( - WeightRenaming, - get_checkpoint_conversion_mapping, - get_model_conversion_mapping, -) - -from vllm.compilation.decorators import DynamicArgDims, support_torch_compile -from vllm.config.utils import getattr_iter -from vllm.distributed import get_pp_group, get_tp_group -from vllm.distributed.utils import get_pp_indices -from vllm.logger import init_logger -from vllm.model_executor.layers.attention import Attention, EncoderOnlyAttention -from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase -from vllm.model_executor.layers.fused_moe import MoERunner -from vllm.model_executor.models.interfaces import ( - SupportsEagle, - SupportsEagle3, - SupportsLoRA, - SupportsPP, - SupportsQuant, -) -from vllm.model_executor.models.interfaces_base import VllmModel -from vllm.model_executor.models.transformers.fuser import BaseFuser, Fusers -from vllm.model_executor.models.transformers.fusers import AttentionFuser, MLAFuser -from vllm.model_executor.models.transformers.fusers.attention import ( - VLLM_ATTN_IMPL, - VLLM_MLA_ATTN_IMPL, -) -from vllm.model_executor.models.transformers.utils import ( - attrsetter, - can_enable_torch_compile, - disable_noop_dynamic_rope, - get_feature_request_tip, - init_on_device_without_buffers, - log_replacement, - named_state, - replace_conv_class, - replace_embedding_class, - replace_layernorm_class, - replace_linear_class, -) -from vllm.model_executor.models.utils import ( - AutoWeightsLoader, - PPMissingLayer, - ShardId, - WeightsMapper, - make_empty_intermediate_tensors_factory, - maybe_prefix, -) -from vllm.sequence import IntermediateTensors - -if TYPE_CHECKING: - from transformers import PreTrainedModel - - from vllm.config import VllmConfig - -try: - from vllm.model_executor.layers.attention.mla_attention import MLAAttention -except ImportError: # MLA support removed from this build - MLAAttention = type("AbsentMLAAttention", (), {}) - -logger = init_logger(__name__) - -VLLM_ATTN_ATTR = "attn" -"""Attribute in which to store the vLLM `Attention` instance that serves a Transformers -attention module. `"attn"` is chosen deliberately to match the mappings generated by -`maybe_remap_kv_scale_name`.""" - - -class PreTrainedModelClasses(NamedTuple): - decoder: type["PreTrainedModel"] - encoders: dict[str, type["PreTrainedModel"]] - """Modality -> encoder class, for each modality that has one.""" - - -class Base( - nn.Module, - VllmModel, - SupportsQuant, - SupportsLoRA, - SupportsPP, - SupportsEagle, - SupportsEagle3, -): - hf_to_vllm_mapper: WeightsMapper - - # TODO transformers will have a util to get these - embedding_modules = {"embed_tokens": "input_embeddings"} - - def __init__(self, *, vllm_config: "VllmConfig", prefix: str = ""): - nn.Module.__init__(self) - logger.info("Using Transformers modeling backend.") - - self.vllm_config = vllm_config - self.config = vllm_config.model_config.hf_config - self.text_config = self.config.get_text_config() - self.cache_config = vllm_config.cache_config - self.compilation_config = vllm_config.compilation_config - self.device_config = vllm_config.device_config - self.model_config = vllm_config.model_config - self.parallel_config = vllm_config.parallel_config - self.quant_config = vllm_config.quant_config - - self.pp_group = get_pp_group() - self.tp_group = get_tp_group() - - # Attrs for weight loading (see self.load_weights) - self.ignore_unexpected_prefixes: list[str] = [] - """Ignore unexpected weights whose qualname starts with these prefixes.""" - self.ignore_unexpected_suffixes: list[str] = [] - """Ignore unexpected weights whose qualname ends with these suffixes.""" - self.packed_modules_mapping: dict[str, list[str]] = {} - """Fused module -> constituent projections, populated by `recursive_replace` - for the quantization machinery and loaders (e.g. bitsandbytes).""" - self.fusers: dict[str, list[BaseFuser]] = {} - """Module qualname -> the fusers applied to it, populated - by `recursive_replace` for `_create_attention_instances`.""" - self.attention_fusers: dict[int, tuple[str, AttentionFuser]] = {} - """`layer_idx` -> the qualname and fuser of the module computing that - layer's attention, populated by `recursive_replace` for - `_create_attention_instances`.""" - - # Attrs for Eagle3 (see self.set_aux_hidden_state_layers) - self._target_class: type[nn.Module] = nn.Module - """Target class for Eagle3 aux hidden state recording.""" - self._layer_names: dict[int, str] = {} - """Mapping from layer index to layer name for Eagle3.""" - self._output_aux_hidden_states_kwargs: dict[str, bool] = {} - """Kwargs to pass to model forward for Eagle3 aux hidden states.""" - - if self.quant_config: - quant_method_name = self.quant_config.get_name() - # Check for unsupported quantization methods. - if quant_method_name in ("mxfp4", "gpt_oss_mxfp4"): - raise NotImplementedError( - "Transformers modeling backend does " - "not support MXFP4 quantization yet." - ) - - self._patch_config() - self._decorate_for_torch_compile() - # Init on "meta" to delay allocating GPU tensors - with ( - self._mark_model_components(vllm_config), - init_on_device_without_buffers("meta"), - ): - from_config_kwargs = self._from_config_kwargs - self.model: PreTrainedModel = AutoModel.from_config(**from_config_kwargs) - - # Create weight name to module qualname mapper - self._create_hf_to_vllm_mapper() - # Remove layers not on this pipeline parallel rank - self.pipeline_parallel() - # Skip the device sync of dynamic rope where it can never rescale - disable_noop_dynamic_rope(self.model.get_decoder(), vllm_config) - # Substitute remaining layers with vLLM's layers as needed - self.recursive_replace() - # Create attention instances for KV cache allocation - self._create_attention_instances(prefix) - - # Initialize any parameters that have not had their modules replaced - self.init_parameters(self.model) - - # Upcast weights Transformers always keeps in fp32 - self.keep_in_fp32(self.model) - - # Pipeline parallel intermediate tensors - self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory( - ["hidden_states"], self.text_config.hidden_size - ) - - def _patch_config(self): - """Patch the config to ensure that the model is created correctly: - - - Sets the attention implementation to "vllm" so the attention instances from - `_create_attention_instances` are used - - Sets the dtype to the default torch dtype set by vLLM because Transformers - uses the config dtype when creating the model - """ - self.text_config._attn_implementation = VLLM_ATTN_IMPL - self.config.dtype = torch.get_default_dtype() - - @contextmanager - def _mark_model_components(self, vllm_config: "VllmConfig"): - """Mark language model and tower submodules as `self.model` is created. - - Nothing to do in `Base`, `MultiModalMixin` will override.""" - yield - - @cached_property - def _from_config_kwargs(self) -> dict[str, Any]: - """The kwargs used to create `self.model`.""" - return dict( - config=self.config, - dtype=self.model_config.dtype, - trust_remote_code=self.model_config.trust_remote_code, - ) - - def _find_encoder_classes( - self, model: "PreTrainedModel" - ) -> dict[str, type["PreTrainedModel"]]: - """Find the encoder class of each modality `model` has one for. - - Text models have none. Multi-modal ones override this. - """ - return {} - - @cached_property - def _pre_trained_model_classes(self) -> PreTrainedModelClasses: - """The decoder class and the encoder class of each modality that has one. - - Both come from a single throwaway model, since building one is not cheap. - """ - with torch.device("meta"): - model: PreTrainedModel = AutoModel.from_config(**self._from_config_kwargs) - model_classes = PreTrainedModelClasses( - decoder=type(model.get_decoder()), - encoders=self._find_encoder_classes(model), - ) - del model - - logger.debug("Identified model classes as: %s", model_classes) - return model_classes - - def _decorate_cls_for_torch_compile( - self, - cls: type["PreTrainedModel"], - dynamic_arg_dims: DynamicArgDims | None, - enable_if: Callable[["VllmConfig"], bool], - is_encoder: bool, - ): - """Decorate `cls` to indicate to vLLM that it supports torch compile. - - Args: - cls: The PreTrainedModel class to decorate. - dynamic_arg_dims: A mapping from argument name to the dynamic dimensions - of the argument. If None, default dynamic arg dims will be used. See - [`support_torch_compile`][vllm.compilation.decorators.support_torch_compile] - for more details. - enable_if: A function which takes in the vLLM config and returns whether - torch compile should be enabled for this class. - is_encoder: Whether the class being decorated is an encoder. - - """ - logger.debug( - "Decorating `%s` as %s for torch compile with dynamic_arg_dims of %s", - cls.__name__, - "encoder" if is_encoder else "decoder", - dynamic_arg_dims, - ) - - support_torch_compile( - dynamic_arg_dims=dynamic_arg_dims, - enable_if=enable_if, - is_encoder=is_encoder, - use_aot_compile=False, - )(cls) - - def _decorate_for_torch_compile(self): - """Decorate the model's decoder class to indicate to vLLM that it - supports torch compile if `can_enable_torch_compile` is True.""" - self._decorate_cls_for_torch_compile( - cls=self._pre_trained_model_classes.decoder, - # Applied to a PreTrainedModel so the batch dimension will exist - dynamic_arg_dims=DynamicArgDims( - input_ids=1, # shape: [1, seq_len] - inputs_embeds=1, # shape: [1, seq_len, hidden_size] - position_ids=-1, # shape: [1, seq_len] or [3, 1, seq_len] for mrope - ), - enable_if=can_enable_torch_compile, - is_encoder=False, - ) - - def _create_hf_to_vllm_mapper(self): - """Create a WeightsMapper to map checkpoint weight names to module qualnames. - - This handles: - - - Transformers weight renaming from `WeightRenaming` - - Checkpoints saved with a base model prefix that is not `model` - - Checkpoints saved with no base model prefix - - Any quantization config specific mappings - """ - orig_to_new_renaming: list[WeightRenaming] = [] - orig_to_new_regex: dict[re.Pattern, str | None] = {} - - mappings = get_model_conversion_mapping(self.model) - # Ensure layers that are part of the model, but not AutoModel, get their mapping - instantiated = {type(m).__name__ for m in self.model.modules()} - for arch in self.config.architectures or []: - if arch not in instantiated: - mappings.extend(get_checkpoint_conversion_mapping(arch) or []) - - for mapping in mappings: - # Handle weights which have been renamed in Transformers - if isinstance(mapping, WeightRenaming): - orig_to_new_renaming.append(mapping) - # TODO: Handle WeightConverter to enable layer merging - - # Handle unexpected weights which should be ignored - if self.model._keys_to_ignore_on_load_unexpected is not None: - for key in self.model._keys_to_ignore_on_load_unexpected: - orig_to_new_regex[re.compile(key)] = None - - # Standardise base model prefix - bmp = self.model.base_model_prefix - expected_bmp = r"model.\1" - # Handle checkpoints saved with different base model prefix - if bmp and bmp != "model": - different_bmp_pattern = re.compile(rf"^{bmp}\.(.+)") - orig_to_new_regex[different_bmp_pattern] = expected_bmp - # Handle direct children of self.model which were saved without the model prefix - direct_children = chain( - self.model.named_children(), - self.model.named_parameters(recurse=False), - self.model.named_buffers(recurse=False), - ) - model_children = "|".join(name for name, _ in direct_children) - missing_bmp_pattern = re.compile(rf"^(?!model\.)(({model_children}).*)") - orig_to_new_regex[missing_bmp_pattern] = expected_bmp - # Handle weights saved as direct children of self.model which no longer are - unexpected_bmp_pattern = re.compile(rf"^(model\.)((?!{model_children}).+)") - orig_to_new_regex[unexpected_bmp_pattern] = r"\2" - # Handle lm_head which was saved inside the base model - nested_lm_head_pattern = re.compile(r"^model\.(.+\.)*(lm_head.+)") - orig_to_new_regex[nested_lm_head_pattern] = r"\2" - - self.hf_to_vllm_mapper = WeightsMapper( - orig_to_new_renaming=orig_to_new_renaming, - orig_to_new_regex=orig_to_new_regex, - ) - - # Apply mapping to quantization config if needed - self._maybe_apply_model_mapping() - - def _get_tie_word_embeddings(self): - """Check if the model has tied word embeddings.""" - # Models created with Transformers v4 and v5 will store this in different places - tie_word_embeddings_v4 = getattr(self.text_config, "tie_word_embeddings", False) - tie_word_embeddings_v5 = getattr(self.config, "tie_word_embeddings", False) - return tie_word_embeddings_v4 or tie_word_embeddings_v5 - - def pipeline_parallel(self): - """Apply the model's pipeline parallelization plan.""" - if self.pp_group.world_size <= 1: - return - - if self.model.supports_pp_plan: - module = self.model - names = list(module._pp_plan.keys()) - else: - module = self.model.get_decoder() - has_parameters = lambda m: next(m.parameters(), None) is not None - names = [n for n, c in module.named_children() if has_parameters(c)] - tip = get_feature_request_tip( - self.model_config.model, self.model_config.trust_remote_code - ) - logger.warning_once( - "%s does not define a pipeline parallel plan. The Transformers " - "modeling backend will infer the split from the layers of %s in order " - "of declaration and keep parameter-free modules on every rank. This " - "may fail if the model's structure is non-standard. %s", - type(self.model).__name__, - type(module).__name__, - tip, - ) - - module_lists = [] - module_list_idx = None - for i, name in enumerate(names): - # attrgetter in case the module is nested (e.g. "text_model.layers") - if isinstance(attrgetter(name)(module), nn.ModuleList): - module_lists.append(name) - module_list_idx = i - - if len(module_lists) > 1: - raise ValueError( - "Pipeline parallel of models with multiple `ModuleList`s " - "in the base model are not supported yet!" - ) - if module_list_idx is None: - raise ValueError(f"Could not find `ModuleList` in {type(module)}") - - # Layers before module list - for name in names[:module_list_idx]: - if self.pp_group.is_first_rank or ( - self._get_tie_word_embeddings() and self.pp_group.is_last_rank - ): - continue - # attrsetter in case the module is nested (e.g. "text_model.embed_tokens") - attrsetter(name)(module, PPMissingLayer()) - - # Module list - start_layer, end_layer = get_pp_indices( - self.text_config.num_hidden_layers, - self.pp_group.rank_in_group, - self.pp_group.world_size, - ) - layers_name = names[module_list_idx] - # attrgetter in case the module is nested (e.g. "text_model.layers") - layers = attrgetter(layers_name)(module) - for i in range(len(layers)): - if start_layer <= i and i < end_layer: - continue - layers[i] = PPMissingLayer() - - # Layers after module list - for name in names[module_list_idx + 1 :]: - # Modules that should be on last rank - if not self.pp_group.is_last_rank: - # attrsetter in case the module is nested (e.g. "text_model.norm") - attrsetter(name)(module, PPMissingLayer()) - - def _vocab_embeddings(self) -> set[nn.Embedding]: - """The `nn.Embedding`s in `self.model` which hold a vocab table.""" - - def vocab_sizes(config: PreTrainedConfig): - for key, value in vars(config).items(): - if isinstance(value, PreTrainedConfig): - yield from vocab_sizes(value) - elif "vocab_size" in key and isinstance(value, int): - yield value - - sizes = set(vocab_sizes(self.config)) - # `get_input_embeddings` may return a module which composes the `nn.Embedding`, - # or a `PPMissingLayer` if the embeddings are not on this pipeline stage - embeddings = { - module - for module in self.model.get_input_embeddings().modules() - if isinstance(module, nn.Embedding) - } - embeddings.update( - module - for module in self.model.modules() - if isinstance(module, nn.Embedding) and module.num_embeddings in sizes - ) - return embeddings - - def recursive_replace(self): - """Recursively replace modules in the model as needed. - - Currently, this replaces: - - - GLUs with a fused `MergedColumnParallelLinear` + `...AndMul` - - Attention QKV projections with a fused `QKVParallelLinear` + split - - `nn.Linear` with vLLM's tensor parallel linear classes - - `nn.Conv2d` / `nn.Conv3d` with vLLM's `Conv2d` / `Conv3d` - - Vocab `nn.Embedding`s with vLLM's `VocabParallelEmbedding` - - RMSNorm (detected from their dataflow) with vLLM's `RMSNorm`or `GemmaRMSNorm` - - `nn.LayerNorm` with vLLM's `LayerNorm` - """ - tp_plan = self.model.tp_plan or {} - - if not tp_plan and self.tp_group.world_size > 1: - tip = get_feature_request_tip( - self.model_config.model, self.model_config.trust_remote_code - ) - logger.warning_once( - "%s does not define a tensor parallel plan. The Transformers modeling " - "backend will shard the model the best it can during graph fusion and " - "replicate the rest. This may be suboptimal or fail if the model does " - "not fuse cleanly. %s", - type(self.model).__name__, - tip, - ) - - # Prefix the patterns because we always start from `self.model` - tp_plan = {maybe_prefix("model", k): v for k, v in tp_plan.items()} - # Detect fusable patterns once per module class (cached, so this is cheap) - fusers = Fusers(self.model, self.vllm_config) - - vocab_embeddings = self._vocab_embeddings() - - orig_to_new_stacked: dict[str, tuple[str, ShardId]] = {} - - def register_fusion(fuser: BaseFuser, prefix: str, module: nn.Module): - """Register a fused layer's mappings just before it is built.""" - self.fusers.setdefault(prefix, []).append(fuser) - - if ( - isinstance(fuser, AttentionFuser) - and (index := fuser.layer_index(module)) is not None - ): - if index in self.attention_fusers: - other, _ = self.attention_fusers[index] - raise ValueError( - f"Layer {index} dispatches attention from more than one " - f"module ({other} and {prefix}), which one vLLM attention " - "layer cannot serve." - ) - self.attention_fusers[index] = (prefix, fuser) - - orig_to_new_stacked.update(fuser.orig_to_new_stacked(prefix)) - - packed_modules_mapping = fuser.packed_modules_mapping - self.packed_modules_mapping.update(packed_modules_mapping) - if self.quant_config is not None: - self.quant_config.packed_modules_mapping.update(packed_modules_mapping) - - def _recursive_replace(module: nn.Module, prefix: str): - for child_name, child_module in module.named_children(): - new_module = child_module - qual_name = maybe_prefix(prefix, child_name) - if ( - isinstance(module, nn.ModuleList) - and len(module) == self.text_config.num_hidden_layers - ): - # Populate Eagle3 attrs - self._target_class = type(child_module) - layer_name = qual_name.removeprefix("model.") - self._layer_names[int(child_name)] = layer_name - # MTP weights should not be loaded into the base model - num_hidden_layers = self.text_config.num_hidden_layers - names = ( - "n_predict", # Override from SpeculativeConfig - "num_nextn_predict_layers", # Most models - "mtp_num_hidden_layers", # Qwen 3.5 - ) - n_predict = getattr_iter(self.text_config, names, 0) - for i in range(num_hidden_layers, num_hidden_layers + n_predict): - mtp_prefix = f"{prefix}.{i}." - if mtp_prefix not in self.ignore_unexpected_prefixes: - self.ignore_unexpected_prefixes.append(mtp_prefix) - # Replace modules as needed - if isinstance(child_module, nn.Linear): - generator = (p for p in tp_plan if re.match(p, qual_name)) - pattern = next(generator, None) - # Some weight loaders expect all linear layers to inherit - # LinearBase, so we set a default style which causes any - # unspecified layers to be replaced with ReplicatedLinear - style = tp_plan.get(pattern, "replicate") - new_module = replace_linear_class( - child_module, style, self.quant_config, prefix=qual_name - ) - elif isinstance(child_module, (nn.Conv2d, nn.Conv3d)): - new_module = replace_conv_class(child_module) - elif child_module in vocab_embeddings: - new_module = replace_embedding_class( - child_module, self.quant_config, prefix=qual_name - ) - elif isinstance(child_module, nn.LayerNorm): - new_module = replace_layernorm_class(child_module) - elif child_module_fusers := fusers[child_module]: - for fuser in child_module_fusers: - register_fusion(fuser, qual_name, child_module) - new_module = fuser.fuse(new_module, qual_name, self.vllm_config) - logger.info_once(fuser.info(child_name)) - _recursive_replace(new_module, prefix=qual_name) - elif not isinstance(child_module, MoERunner): - # MoERunner can contain aliases of shared experts and gates, - # so we don't want to recurse into it and break weight loading. - _recursive_replace(child_module, prefix=qual_name) - - if new_module is not child_module: - setattr(module, child_name, new_module) - log_replacement(qual_name, child_module, new_module) - - _recursive_replace(self.model, prefix="model") - - self.hf_to_vllm_mapper |= WeightsMapper(orig_to_new_stacked=orig_to_new_stacked) - - def _create_attention_instances(self, prefix: str = ""): - """Create `Attention` instances to inform KV cache allocation.""" - text_config = self.text_config - attn_cls = self._get_attn_cls() - - # kv_lora_rank indicates that this is an MLA model - if getattr(text_config, "kv_lora_rank", None) is not None: - if attn_cls is MLAAttention: - text_config._attn_implementation = VLLM_MLA_ATTN_IMPL - else: - # MLA model not using MLAAttention: recompute head_size for full attn - qk_nope_head_dim = getattr(text_config, "qk_nope_head_dim", 0) - qk_rope_head_dim = getattr(text_config, "qk_rope_head_dim", 0) - if qk_head_dim := qk_nope_head_dim + qk_rope_head_dim: - self.model_config.model_arch_config.head_size = qk_head_dim - - logits_soft_cap = getattr(text_config, "attn_logit_softcapping", None) - layer_types = getattr(text_config, "layer_types", None) - - pp_rank = self.pp_group.rank_in_group - pp_size = self.pp_group.world_size - start, end = get_pp_indices(text_config.num_hidden_layers, pp_rank, pp_size) - - for i in range(start, end): - if i not in self.attention_fusers: - layer = ( - f"{i} ({layer_types[i]})" - if layer_types and i < len(layer_types) - else str(i) - ) - raise ValueError( - f"Layer {layer} does not dispatch through the Transformers " - "attention interface and vLLM has no other way to handle it." - ) - attn_prefix, attn_fuser = self.attention_fusers[i] - attn_module = self.get_submodule(attn_prefix) - - # `[i]` is the whole-model config unless the checkpoint is - # heterogeneous, in which case it is this layer's own geometry. - arch_config = self.model_config.model_arch_config[i] - num_heads = self.model_config.get_num_attention_heads( - self.parallel_config, arch_config - ) - head_size = arch_config.head_size - if (scale := attn_fuser.scale(attn_module)) is None: - # Default to Llama scale if AttentionFuser couldn't identify it - scale = head_size**-0.5 - - kwargs = dict( - num_heads=num_heads, - scale=scale, - cache_config=self.cache_config, - quant_config=self.quant_config, - prefix=maybe_prefix(prefix, f"{attn_prefix}.{VLLM_ATTN_ATTR}"), - ) - - if attn_cls is MLAAttention: - mla_fuser = next( - (f for f in self.fusers[attn_prefix] if isinstance(f, MLAFuser)), - None, - ) - if mla_fuser is None: - raise ValueError( - f"Layer {i} ({attn_prefix}) did not fuse as MLA, but the rest " - "of the model did, so its latent KV cache cannot be allocated." - ) - from vllm.model_executor.layers.attention.mla_attention import ( - get_mla_dims, - ) - - dims = get_mla_dims(self.model_config) - kwargs.update( - qk_nope_head_dim=dims.qk_nope_head_dim, - qk_rope_head_dim=dims.qk_rope_head_dim, - v_head_dim=dims.v_head_dim, - q_lora_rank=dims.q_lora_rank, - kv_lora_rank=dims.kv_lora_rank, - kv_b_proj=attn_module.get_submodule(mla_fuser.kv_b_proj_name), - ) - else: - kwargs.update( - head_size=head_size, - num_kv_heads=self.model_config.get_num_kv_heads( - self.parallel_config, arch_config - ), - logits_soft_cap=logits_soft_cap, - ) - - # Handle interleaved sliding window attention - if layer_types and layer_types[i] == "sliding_attention": - kwargs["per_layer_sliding_window"] = text_config.sliding_window - # Handle attention sinks - if (sinks := attn_fuser.sinks(attn_module)) is not None: - kwargs["sinks"] = sinks - - attn_instance = attn_cls(**kwargs) - - # Attach the instance to the module it serves, which is the only way it - # is in the module tree at all: a plain dict is invisible to - # `nn.Module.__setattr__`. Being in the tree runs its - # `process_weights_after_loading` and moves its KV-scale buffers with - # `model.to(device)`, and lets the attention interface read it off the - # module rather than index by `module.layer_idx`, an `int` that bakes - # layer identity into the traced graph and so costs one compiled - # artifact per layer. - setattr(attn_module, VLLM_ATTN_ATTR, attn_instance) - - def _get_attn_cls(self) -> type[AttentionLayerBase]: - """Return the `Attention` class to use for this model's layers.""" - # In encoder models, the attention layers will have `is_causal=False` - is_encoder = lambda module: not getattr(module, "is_causal", True) - has_encoder = lambda model: any(is_encoder(m) for m in model.modules()) - is_multimodal = lambda config: config != config.get_text_config() - # vLLM does not support encoder-decoder models, so if any encoder layer is - # found in a text only model, we assume the whole model is an encoder model - if has_encoder(self.model) and not is_multimodal(self.config): - return EncoderOnlyAttention - if self.model_config.use_mla: - if any( - isinstance(fuser, MLAFuser) - for fusers in self.fusers.values() - for fuser in fusers - ): - return MLAAttention - logger.warning_once( - "This model uses MLA but `MLAFuser` failed to match and/or fuse any " - "MLA attention layers. Falling back to full attention with a padded " - "`value` head dimension." - ) - os.environ["VLLM_MLA_DISABLE"] = "1" - return Attention - - def init_parameters(self, module: nn.Module, dtype: torch.dtype | None = None): - """If a `parameter` is on the `meta` device, then its parent - `module` is the original module created by: - - ```python - with torch.device("meta"): - self.model: "PreTrainedModel" = AutoModel.from_config(...) - ``` - """ - dtype = dtype or self.model_config.dtype - device = self.device_config.device - - def _init_parameters(module: nn.Module): - for name, param in module.named_parameters(recurse=False): - # Already on device, nothing to do - if param.device != torch.device("meta"): - continue - # Already a vLLM parameter, nothing to do - if hasattr(param, "weight_loader"): - continue - data = torch.empty_like(param.data, dtype=dtype, device=device) - setattr(module, name, nn.Parameter(data=data)) - for child in module.children(): - _init_parameters(child) - - _init_parameters(module) - - def keep_in_fp32(self, module: nn.Module): - """Honor `_keep_in_fp32_modules_strict` as `from_pretrained` would.""" - if self.model_config.dtype not in (torch.float16, torch.bfloat16): - return - fragments = getattr(module, "_keep_in_fp32_modules_strict", None) - if not fragments: - return - pattern = re.compile("|".join(re.escape(f) for f in fragments)) - for name, tensor in named_state(module): - if not hasattr(tensor, "weight_loader") and pattern.search(name): - tensor.data = tensor.data.to(torch.float32) - - def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: - return self.model.get_input_embeddings()(input_ids) - - def forward( - self, - input_ids: torch.Tensor | None, - positions: torch.Tensor, - intermediate_tensors: IntermediateTensors | None = None, - inputs_embeds: torch.Tensor | None = None, - **kwargs, - ) -> torch.Tensor | IntermediateTensors: - if not self.pp_group.is_first_rank: - assert intermediate_tensors is not None - input_ids = None - inputs_embeds = intermediate_tensors["hidden_states"] - - # Add batch dimension before entering Transformers model - if input_ids is not None and input_ids.ndim == 1: - # [seq_len] -> [1, seq_len] - input_ids = input_ids[None, ...] - if inputs_embeds is not None and inputs_embeds.ndim == 2: - # [seq_len, hidden_size] -> [1, seq_len, hidden_size] - inputs_embeds = inputs_embeds[None, ...] - if positions.ndim == 1: - # [seq_len] -> [1, seq_len] - positions = positions[None, ...] - - # Transformers models expect either input_ids or inputs_embeds, but not both - if input_ids is not None and inputs_embeds is not None: - input_ids = None - - outputs = self.model( - input_ids=input_ids, - inputs_embeds=inputs_embeds, - use_cache=False, - position_ids=positions, - return_dict=False, - **self._output_aux_hidden_states_kwargs, - **kwargs, - ) - - # Remove batch dimension after exiting Transformers model - hidden_states = outputs[0][0, ...] - if self._output_aux_hidden_states_kwargs: - aux_hidden_states = [x[0][0, ...] for x in outputs[1:]] - - if not self.pp_group.is_last_rank: - return IntermediateTensors({"hidden_states": hidden_states}) - - if self._output_aux_hidden_states_kwargs and len(aux_hidden_states) > 0: - return hidden_states, aux_hidden_states - return hidden_states - - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - loader = AutoWeightsLoader( - self, - ignore_unexpected_prefixes=self.ignore_unexpected_prefixes, - ignore_unexpected_suffixes=self.ignore_unexpected_suffixes, - ) - return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) - - @staticmethod - def check_version(min_version: str, feature: str): - installed = Version(transformers.__version__) - required = Version(min_version) - if installed < required: - raise ImportError( - f"Transformers modeling backend requires transformers>={required} " - f"for {feature}, but got {installed}" - ) - - def set_aux_hidden_state_layers(self, layers: tuple[int, ...]) -> None: - from transformers.utils.output_capturing import ( - OutputRecorder, - maybe_install_capturing_hooks, - ) - - # The default value in PreTrainedModel is None - if self.model._can_record_outputs is None: - self.model._can_record_outputs = {} - - target_class = self._target_class - for layer in layers: - # layer - 1 because we want the input to the layer - layer_name = self._layer_names[layer - 1] - layer_key = f"aux_hidden_state_{layer}" - aux_hidden_state_i = OutputRecorder(target_class, layer_name=layer_name) - self.model._can_record_outputs[layer_key] = aux_hidden_state_i - self._output_aux_hidden_states_kwargs[f"output_{layer_key}"] = True - - # Ensure that the capture hooks are installed before dynamo traces the model - maybe_install_capturing_hooks(self.model) - - def get_eagle3_default_aux_hidden_state_layers(self) -> tuple[int, ...]: - num_layers = self.text_config.num_hidden_layers - return (2, num_layers // 2, num_layers - 3) diff --git a/vllm/model_executor/models/transformers/causal.py b/vllm/model_executor/models/transformers/causal.py deleted file mode 100644 index 9124784d36..0000000000 --- a/vllm/model_executor/models/transformers/causal.py +++ /dev/null @@ -1,89 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -# Copyright 2024 The vLLM team. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Transformers modeling backend mixin for causal language models.""" - -from collections.abc import Iterable -from typing import TYPE_CHECKING - -import torch - -from vllm.model_executor.layers.logits_processor import LogitsProcessor -from vllm.model_executor.layers.vocab_parallel_embedding import ( - ParallelLMHead, - VocabParallelEmbedding, -) -from vllm.model_executor.models.interfaces_base import VllmModelForTextGeneration -from vllm.model_executor.models.utils import PPMissingLayer, maybe_prefix - -from .base import Base - -if TYPE_CHECKING: - from vllm.config import VllmConfig - - -class CausalMixin(VllmModelForTextGeneration, Base): - def __init__(self, *, vllm_config: "VllmConfig", prefix: str = ""): - # Skip VllmModelForTextGeneration.__init__ and call the next class in MRO - super(VllmModelForTextGeneration, self).__init__( - vllm_config=vllm_config, prefix=prefix - ) - - tie_word_embeddings = self._get_tie_word_embeddings() - - if self.pp_group.is_last_rank: - self.lm_head = ParallelLMHead( - self.text_config.vocab_size, - self.text_config.hidden_size, - quant_config=self.quant_config, - prefix=maybe_prefix(prefix, "lm_head"), - ) - if tie_word_embeddings: - for module in self.model.get_input_embeddings().modules(): - if isinstance(module, VocabParallelEmbedding): - self.lm_head = self.lm_head.tie_weights(module) - break - - self.logits_processor = LogitsProcessor( - self.text_config.vocab_size, - scale=getattr(self.text_config, "logit_scale", 1.0), - soft_cap=getattr(self.text_config, "final_logit_softcapping", None), - ) - else: - self.lm_head = PPMissingLayer() - - def load_weights(self, weights: Iterable[tuple[str, "torch.Tensor"]]) -> set[str]: - """A thin wrapper around `Base.load_weights` to handle the lm_head bias.""" - lm_head_bias = set() - load_device = self.vllm_config.load_config.device or self.device_config.device - - def auto_load_lm_head_bias(weights): - for name, weight in weights: - if name.endswith("lm_head.bias") and self.pp_group.is_last_rank: - # load_weights is called outside vLLM's device context, - # so we must enter it again when registering the bias. - with torch.device(load_device): - self.lm_head._register_bias() - self.lm_head.bias.weight_loader(self.lm_head.bias, weight) - lm_head_bias.add(name) - else: - yield name, weight - - return super().load_weights(auto_load_lm_head_bias(weights)) | lm_head_bias - - def compute_logits(self, hidden_states: "torch.Tensor") -> "torch.Tensor | None": - logits = self.logits_processor(self.lm_head, hidden_states, self.lm_head.bias) - return logits diff --git a/vllm/model_executor/models/transformers/fuser.py b/vllm/model_executor/models/transformers/fuser.py deleted file mode 100644 index 91c232f6d5..0000000000 --- a/vllm/model_executor/models/transformers/fuser.py +++ /dev/null @@ -1,126 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Fuser detection for the Transformers modeling backend.""" - -from collections import UserDict -from typing import TYPE_CHECKING, TypeVar - -from cachetools import cached -from torch import nn - -from vllm.logger import init_logger -from vllm.model_executor.models.transformers.fusers import ( - AttentionFuser, - BaseFuser, - GLUFuser, - MergedColumnParallelFuser, - MLAFuser, - PackedQKVFuser, - QKVFuser, - RewriteFuser, - RMSNormFuser, -) -from vllm.model_executor.models.transformers.fx_utils import trace - -if TYPE_CHECKING: - from vllm.config import VllmConfig - -logger = init_logger(__name__) - -F = TypeVar("F", bound=BaseFuser) - - -def key(module: nn.Module) -> tuple: - """Cache key for `get_fusers`. Considers module type and its immediate children.""" - return (type(module), tuple(name for name, _ in module.named_children())) - - -FUSERS: tuple[type[BaseFuser], ...] = ( - # Order these by priority, e.g. QKV would be preferred over MergedColumnParallel - MLAFuser, - GLUFuser, - QKVFuser, - PackedQKVFuser, - MergedColumnParallelFuser, - RMSNormFuser, - # Put fusers that don't redefine forward after this comment - AttentionFuser, -) -"""Every fuser, in priority order: those that redefine the forward first, then those -that leave it alone. A new fuser is added here.""" - - -@cached(cache={}, key=key) -def get_fusers(module: nn.Module) -> list[BaseFuser]: - """Every fuser that could apply to `module`'s class (cached), in `FUSERS` order.""" - # Projection fusions need >=2 sibling linears; the RMSNorm fusion needs a - # leaf module (raw tensor math, no submodules). Nothing else can match, and - # tracing is skipped for it. - n_linear = sum(isinstance(c, nn.Linear) for c in module.children()) - is_leaf = next(module.children(), None) is None - graph = trace(module) if n_linear >= 2 or is_leaf else None - - # Every redefining candidate that matches is cached, not just the first: - # instances of this class can be shaped heterogeneously, so which one is - # usable is decided per instance in Fusers.__getitem__, not here. A forward - # can only be rewritten from a trace of it, so each starts fresh regardless. - fusers: list[BaseFuser] = [] - for fuser_cls in FUSERS: - if fuser_cls.redefines_forward and graph is None: - continue - if (fuser := fuser_cls.match(graph, module)) is None: - continue - if isinstance(fuser, RewriteFuser): - try: - fuser.update_forward(module) - except Exception as exc: - logger.debug( - "Attempted to fuse %s using %s but failed " - "to update its forward method: %s", - type(module), - fuser_cls.__name__, - exc, - ) - continue - fusers.append(fuser) - - # A norm we could not match structurally is left unfused; flag likely misses. - if ( - module.__class__.__name__.endswith("RMSNorm") - and graph is not None - and not any(isinstance(fuser, RMSNormFuser) for fuser in fusers) - ): - logger.warning_once( - "%s looks like an RMSNorm but its computation did not match the " - "expected pattern, so it was left unfused.", - module.__class__.__name__, - ) - return fusers - - -def get_fuser(module: nn.Module, fuser_cls: type[F]) -> "F | None": - """The first `fuser_cls` that applies to `module`, if one does.""" - return next((f for f in get_fusers(module) if isinstance(f, fuser_cls)), None) - - -class Fusers(UserDict): - """Mapping from module class and shape to fusers, for all fusable modules.""" - - def __init__(self, model: nn.Module, vllm_config: "VllmConfig"): - self.vllm_config = vllm_config - super().__init__({key(m): get_fusers(m) for m in model.modules()}) - - def __getitem__(self, m: nn.Module) -> list[BaseFuser]: - """The fusers this instance can take, in the order to apply them.""" - chosen: list[BaseFuser] = [] - redefined = False - for fuser in self.data.get(key(m), ()): - # Only one candidate may rewrite m's forward; take the first - # (highest-priority) one that validates for this instance. - if fuser.redefines_forward and redefined: - continue - if not fuser.validate(m, self.vllm_config): - continue - redefined = redefined or fuser.redefines_forward - chosen.append(fuser) - return chosen diff --git a/vllm/model_executor/models/transformers/fusers/__init__.py b/vllm/model_executor/models/transformers/fusers/__init__.py deleted file mode 100644 index 727dc0635f..0000000000 --- a/vllm/model_executor/models/transformers/fusers/__init__.py +++ /dev/null @@ -1,33 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Concrete fusers for the Transformers modeling backend.""" - -from vllm.model_executor.models.transformers.fusers.attention import AttentionFuser -from vllm.model_executor.models.transformers.fusers.base import ( - BaseFuser, - RewriteFuser, - StackedFuser, -) -from vllm.model_executor.models.transformers.fusers.glu import GLUFuser -from vllm.model_executor.models.transformers.fusers.merged_column import ( - MergedColumnParallelFuser, -) -from vllm.model_executor.models.transformers.fusers.mla import MLAFuser -from vllm.model_executor.models.transformers.fusers.moe import MoEBlockFuser -from vllm.model_executor.models.transformers.fusers.packed_qkv import PackedQKVFuser -from vllm.model_executor.models.transformers.fusers.qkv import QKVFuser -from vllm.model_executor.models.transformers.fusers.rms_norm import RMSNormFuser - -__all__ = [ - "AttentionFuser", - "BaseFuser", - "RewriteFuser", - "StackedFuser", - "GLUFuser", - "MergedColumnParallelFuser", - "MLAFuser", - "MoEBlockFuser", - "PackedQKVFuser", - "QKVFuser", - "RMSNormFuser", -] diff --git a/vllm/model_executor/models/transformers/fusers/attention.py b/vllm/model_executor/models/transformers/fusers/attention.py deleted file mode 100644 index d8d324536d..0000000000 --- a/vllm/model_executor/models/transformers/fusers/attention.py +++ /dev/null @@ -1,162 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Attention fuser: the module that dispatches to the attention interface.""" - -import ast -import inspect -import textwrap -from collections.abc import Callable -from dataclasses import dataclass -from functools import cache -from typing import TYPE_CHECKING, ClassVar - -import torch -from torch import fx, nn - -from vllm.model_executor.model_loader.weight_utils import sharded_weight_loader -from vllm.model_executor.models.transformers.fusers.base import BaseFuser -from vllm.model_executor.utils import set_weight_attrs - -if TYPE_CHECKING: - from vllm.config import VllmConfig - -VLLM_ATTN_IMPL = "vllm" -VLLM_MLA_ATTN_IMPL = "vllm_mla" - - -def _is_interface_lookup(node: ast.expr | None) -> bool: - """Whether `node` reads an entry out of `ALL_ATTENTION_FUNCTIONS`.""" - # ALL_ATTENTION_FUNCTIONS.get_interface(...) or ALL_ATTENTION_FUNCTIONS[...] - if isinstance(node, ast.Call): - node = node.func - if not isinstance(node, (ast.Attribute, ast.Subscript)): - return False - return ( - isinstance(node.value, ast.Name) and node.value.id == "ALL_ATTENTION_FUNCTIONS" - ) - - -@cache -def interface_call(forward: Callable) -> ast.Call | None: - """The attention interface call in `forward`, if it makes exactly one.""" - try: - source = inspect.getsource(inspect.unwrap(forward)) - tree = ast.parse(textwrap.dedent(source)) - except (AttributeError, OSError, SyntaxError, TypeError): - return None - - # Find the names of the local variables that read from the interface lookup - names: set[str] = set() - for node in ast.walk(tree): - if not isinstance(node, (ast.Assign, ast.AnnAssign)): - continue - if not _is_interface_lookup(node.value): - continue - targets = node.targets if isinstance(node, ast.Assign) else [node.target] - names.update(t.id for t in targets if isinstance(t, ast.Name)) - # Find the calls to those names, and return the one if there is exactly one - calls = [ - node - for node in ast.walk(tree) - if isinstance(node, ast.Call) - and isinstance(node.func, ast.Name) - and node.func.id in names - ] - return calls[0] if len(calls) == 1 else None - - -def _resolve(node: ast.expr, module: nn.Module) -> object: - """The value of `node` on `module`, for literals and `self.`.""" - if isinstance(node, ast.Constant): - return node.value - if ( - isinstance(node, ast.Attribute) - and isinstance(node.value, ast.Name) - and node.value.id == "self" - ): - return getattr(module, node.attr, None) - return None - - -@dataclass -class AttentionFuser(BaseFuser): - """A module that dispatches through the Transformers attention interface.""" - - redefines_forward: ClassVar[bool] = False - - source_cls: str - """Class of the HF module that dispatches (for logging).""" - scale_expr: ast.expr | None = None - """Source of the `scaling=` the module hands the interface, if it hands one.""" - s_aux_expr: ast.expr | None = None - """Source of the `s_aux=` the module hands the interface, if it hands one.""" - - def info(self, name: str) -> str: - return f"Found: {name} ({self.source_cls}) -> attention interface" - - @classmethod - def match( - cls, graph: fx.Graph | None, module: nn.Module - ) -> "AttentionFuser | None": - if (call := interface_call(type(module).forward)) is None: - return None - scaling = [kw.value for kw in call.keywords if kw.arg == "scaling"] - scale_expr = scaling[0] if len(scaling) == 1 else None - s_aux = [kw.value for kw in call.keywords if kw.arg == "s_aux"] - s_aux_expr = s_aux[0] if len(s_aux) == 1 else None - return cls( - source_cls=type(module).__name__, - scale_expr=scale_expr, - s_aux_expr=s_aux_expr, - ) - - def validate(self, module: nn.Module, vllm_config: "VllmConfig") -> bool: - """Whether `module` will actually dispatch to vLLM.""" - config = getattr(module, "config", None) - # Only patched in the text config, this excludes attention based mm encoders - vllm_attn_impls = {VLLM_ATTN_IMPL, VLLM_MLA_ATTN_IMPL} - return getattr(config, "_attn_implementation", None) in vllm_attn_impls - - def fuse( - self, module: nn.Module, prefix: str, vllm_config: "VllmConfig" - ) -> nn.Module: - if (sinks := self.sinks(module)) is not None: - assert isinstance(self.s_aux_expr, ast.Attribute) - size = sinks.numel() // vllm_config.parallel_config.tensor_parallel_size - device = vllm_config.device_config.device - data = torch.empty(size, dtype=sinks.dtype, device=device) - sinks_param = nn.Parameter(data, requires_grad=False) - set_weight_attrs(sinks_param, {"weight_loader": sharded_weight_loader(0)}) - setattr(module, self.s_aux_expr.attr, sinks_param) - return module - - def layer_index(self, module: nn.Module) -> int | None: - """The layer `module` computes attention for, if it declares one.""" - layer_idx = getattr(module, "layer_idx", None) - return layer_idx if isinstance(layer_idx, int) else None - - def scale(self, module: nn.Module) -> float | None: - """The softmax scale `module` passes to the interface, or `None`.""" - if self.scale_expr is None: - return None - scale = _resolve(self.scale_expr, module) - if not isinstance(scale, (int, float)) or isinstance(scale, bool): - expression = ast.unparse(self.scale_expr) - raise ValueError( - f"Cannot resolve attention scaling expression {expression!r} in " - f"{type(module).__name__}." - ) - return float(scale) - - def sinks(self, module: nn.Module) -> nn.Parameter | None: - """The per-head sink tensor `module` passes to the interface, or `None`.""" - if self.s_aux_expr is None: - return None - s_aux = _resolve(self.s_aux_expr, module) - if not isinstance(s_aux, (nn.Parameter, type(None))): - expression = ast.unparse(self.s_aux_expr) - raise ValueError( - f"Cannot resolve attention s_aux expression {expression!r} in " - f"{type(module).__name__}." - ) - return s_aux diff --git a/vllm/model_executor/models/transformers/fusers/base.py b/vllm/model_executor/models/transformers/fusers/base.py deleted file mode 100644 index 937b70c5ed..0000000000 --- a/vllm/model_executor/models/transformers/fusers/base.py +++ /dev/null @@ -1,250 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Base classes for the Transformers backend fusers.""" - -import ast -import types -from abc import ABC, abstractmethod -from collections.abc import Callable, Iterable -from dataclasses import dataclass, field -from typing import TYPE_CHECKING, ClassVar - -from torch import fx, nn - -from vllm.model_executor.models.transformers.fx_utils import ( - aliasing_names, - bypass_existence_guard, - replace_expr, - self_call_and_refs, - written_names, -) -from vllm.model_executor.models.utils import ShardId, maybe_prefix - -if TYPE_CHECKING: - from vllm.config import VllmConfig - - -@dataclass -class BaseFuser(ABC): - """A detected fusion and how to apply it. - - `match` analyses the module *class* once (cached, see `get_fusers`); `fuse` - then applies the fusion to an instance in `recursive_replace`, returning the - module to install in its place. - """ - - redefines_forward: ClassVar[bool] = True - """Whether `fuse` gives the module a different forward, - by rewriting its source or by returning a different module in its place.""" - - @abstractmethod - def info(self, name: str) -> str: - """A human-readable description of the fusion at `name`, for logging.""" - - @classmethod - @abstractmethod - def match(cls, graph: fx.Graph, module: nn.Module) -> "BaseFuser | None": - """Match the pattern in `graph`, returning a fuser if found.""" - - @abstractmethod - def validate(self, module: nn.Module, vllm_config: "VllmConfig") -> bool: - """Whether this fuser can be applied to this `module` instance.""" - - @abstractmethod - def fuse( - self, module: nn.Module, prefix: str, vllm_config: "VllmConfig" - ) -> nn.Module: - """Apply the fusion to an already-validated `module`, returning the - module to install in its place (mutated in place, or freshly built).""" - - def orig_to_new_stacked(self, prefix: str) -> dict[str, tuple[str, ShardId]]: - """`WeightsMapper.orig_to_new_stacked` entries this fuser contributes - (none unless it stacks weights).""" - return {} - - @property - def packed_modules_mapping(self) -> dict[str, list[str]]: - """`packed_modules_mapping` entries this fuser contributes (none unless - it stacks weights).""" - return {} - - -def fused_head_size(module: nn.Module, vllm_config: "VllmConfig") -> int: - """The head size of `module`, which head counts are derived from. - - Prefer the module's own `head_dim`, which Transformers sets per instance: - the model-wide value is the largest head size across layers, so on a - heterogeneous checkpoint it would divide out the wrong number of heads, and - it is the text head size, so it does not describe a vision tower at all. - """ - return getattr(module, "head_dim", None) or vllm_config.model_config.get_head_size() - - -def local_output_sizes(merged_name: str) -> str: - """Source for the per-rank widths of the merged linear `self.`.""" - merged = f"self.{merged_name}" - return f"[s // {merged}.tp_size for s in {merged}.output_sizes]" - - -@dataclass -class RewriteFuser(BaseFuser): - """A fuser that rewrites the module's forward and rebinds it. - - `match` and `update_forward` analyse the class once; `fuse` swaps the - submodules and binds the compiled forward on an instance in place, so it - keeps its class and any attribute the fusion does not consume. - """ - - source_cls: str - """Class of the HF module the fused projections belonged to (for logging).""" - - fused_forward: Callable = field(init=False, repr=False) - """The compiled rewritten forward, set by `update_forward`.""" - - @abstractmethod - def update_forward(self, module: nn.Module) -> None: - """Rewrite and compile `type(module)`'s forward source. - - Raises if the source does not admit the rewrite (fusion is then skipped). - """ - - @abstractmethod - def update_attrs( - self, module: nn.Module, prefix: str, vllm_config: "VllmConfig" - ) -> None: - """Replace `module`'s submodules with their vLLM equivalents.""" - - def fuse( - self, module: nn.Module, prefix: str, vllm_config: "VllmConfig" - ) -> nn.Module: - """Fuse an already-validated `module` in place (see `Fusers.__getitem__`). - - Builds the merged submodule and binds the compiled forward.""" - self.update_attrs(module, prefix, vllm_config) - module.forward = types.MethodType(self.fused_forward, module) - return module - - -@dataclass -class StackedFuser(RewriteFuser): - """A fuser that merges sibling projections into one stacked linear and - rewrites the forward to call it.""" - - merged_cls_name: ClassVar[str] - """Name of the vLLM class the merged projection becomes (for logging).""" - - @property - @abstractmethod - def merged_name(self) -> str: - """Attribute name of the merged module created by `update_attrs`.""" - - def info(self, name: str) -> str: - sources = " + ".join(shard for shard, _ in self.shards) - return ( - f"Fused: {sources} ({name}: {self.source_cls}) -> " - f"{self.merged_name} ({self.merged_cls_name})" - ) - - @property - @abstractmethod - def shards(self) -> list[tuple[str, ShardId]]: - """Each projection's original name and its shard id in the merged module. - - Source for both `orig_to_new_stacked` and `packed_modules_mapping`.""" - - def orig_to_new_stacked(self, prefix: str) -> dict[str, tuple[str, ShardId]]: - """`WeightsMapper.orig_to_new_stacked` entries for one fused instance. - - Maps each checkpoint name to `(merged_name, shard_id)`, keyed by qualname - so only this exact layer is remapped, never a same-named projection - elsewhere (e.g. an unfused MoE expert's `gate_proj`).""" - merged = maybe_prefix(prefix, self.merged_name) - return { - maybe_prefix(prefix, name): (merged, shard) for name, shard in self.shards - } - - @property - def packed_modules_mapping(self) -> dict[str, list[str]]: - """`{merged_name: [projection names]}` so quantization can unpack the - fused layer into its per-shard configs.""" - return {self.merged_name: [name for name, _ in self.shards]} - - def _unguarded_calls( - self, funcdef: ast.FunctionDef, names: Iterable[str] - ) -> list[ast.Call]: - """One `self.(arg)` call per projection, existence guards folded. - - `update_attrs` deletes the projections it merges, so any reference to one beyond - its call site must be a guard on its existence; folding those to their constant - value keeps the rewritten forward off a name that no longer exists. A reference - that is not such a guard raises, so fusion is skipped.""" - calls = [] - for name in names: - call, refs = self_call_and_refs(funcdef, name) - for ref in refs: - bypass_existence_guard(funcdef, ref, name) - calls.append(call) - return calls - - def _check_input_stable( - self, - funcdef: ast.FunctionDef, - module: nn.Module, - calls: list[ast.Call], - block: list[ast.stmt], - indices: list[int], - ) -> None: - """Raise unless hoisting the merged GEMM preserves what it reads. - - Fusing moves every projection to one call at `min(indices)`, so the - merged GEMM reads the input once, up front, where the last of `calls` - would have read it later. That holds only if nothing in between changes - the input, and a change need not name it: writing any view that shares - its storage changes it too. Both halves of the check over-approximate, - since a false hit costs a fusion while a miss returns wrong numbers. - """ - arg_names = { - node.id - for node in ast.walk(calls[0].args[0]) - if isinstance(node, ast.Name) and isinstance(node.ctx, ast.Load) - } - # Protect the names the input reads and anything that may alias them. - tracked = aliasing_names(funcdef, arg_names, module) - # Any of them rebound, written through, or passed to a call that writes - # its argument would leave the hoisted GEMM reading a different value. - region = block[min(indices) : max(indices) + 1] - if tracked & written_names(region): - raise ValueError("projection input is rebound or mutated before all calls") - - def _splice_merged_split( - self, - funcdef: ast.FunctionDef, - calls: list[ast.Call], - block: list[ast.stmt], - index: int, - ) -> None: - """Insert `temps = self.(arg).split(sizes, -1)` at - `block[index]` and replace each of `calls` with its temp name. - - `calls` must share one input argument; `block[index]` must be where - they are (or would be) evaluated. Raises if a generated temporary - would shadow an existing name in `funcdef`. - """ - temps = [f"_vllm_merged_{i}" for i in range(len(calls))] - names = {node.id for node in ast.walk(funcdef) if isinstance(node, ast.Name)} - if names & set(temps): - raise ValueError("fused temporaries would shadow existing names") - targets = ", ".join(temps) - sections = local_output_sizes(self.merged_name) - source = f"{targets} = self.{self.merged_name}(__arg__).split({sections}, -1)" - assign = ast.parse(source).body[0] - arg = next( - node - for node in ast.walk(assign) - if isinstance(node, ast.Name) and node.id == "__arg__" - ) - replace_expr(assign, arg, calls[0].args[0]) - ast.copy_location(assign, block[index]) - block.insert(index, assign) - for call, temp in zip(calls, temps): - replace_expr(funcdef, call, ast.Name(id=temp, ctx=ast.Load())) diff --git a/vllm/model_executor/models/transformers/fusers/glu.py b/vllm/model_executor/models/transformers/fusers/glu.py deleted file mode 100644 index 15d886c473..0000000000 --- a/vllm/model_executor/models/transformers/fusers/glu.py +++ /dev/null @@ -1,191 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""GLU projection fuser: `act(gate(x)) * up(x)` -> a fused gate/up linear.""" - -import ast -import operator -from dataclasses import dataclass -from typing import TYPE_CHECKING, ClassVar - -from torch import fx, nn -from transformers.activations import ACT2CLS - -from vllm.logger import init_logger -from vllm.model_executor.layers.activation import _ACTIVATION_AND_MUL_REGISTRY -from vllm.model_executor.models.transformers.fusers.merged_column import ( - MergedColumnParallelFuser, -) -from vllm.model_executor.models.transformers.fx_utils import ( - compile_forward, - find_node, - is_linear, - peel, - recover_forward, - replace_expr, - single_self_call, -) -from vllm.model_executor.models.transformers.layers import get_act_and_mul_fn -from vllm.model_executor.models.transformers.utils import ( - log_replacement, - replace_linear_class, -) -from vllm.model_executor.models.utils import maybe_prefix - -if TYPE_CHECKING: - from vllm.config import VllmConfig - -logger = init_logger(__name__) - - -CLS2ACT: dict[type, list[str]] = {} -for _act_name, _act_cls in ACT2CLS.items(): - if isinstance(_act_cls, tuple): - _act_cls = _act_cls[0] - CLS2ACT.setdefault(_act_cls, []).append(_act_name) - -ACT_AND_MUL_NAMES = frozenset(_ACTIVATION_AND_MUL_REGISTRY.keys()) - - -@dataclass -class GLUFuser(MergedColumnParallelFuser): - """Fuser for the GLU pattern `act(gate(x)) * up(x)`.""" - - act_name: str - down_name: str | None - merged_name: ClassVar[str] = "gate_up_proj" - - @property - def gate_name(self) -> str: - return self.linear_names[0] - - @property - def up_name(self) -> str: - return self.linear_names[1] - - @classmethod - def _is_act_of_gate(cls, node: fx.Node, module: nn.Module) -> bool: - """Is node `act(gate(x))` where `gate` is linear and `act` is not linear.""" - return ( - node.op == "call_module" - and not is_linear(node, module) - and len(node.args) == 1 - and isinstance(node.args[0], fx.Node) - and is_linear(node.args[0], module) - ) - - @classmethod - def _get_glu_nodes( - cls, graph: fx.Graph, module: nn.Module - ) -> tuple[fx.Node, fx.Node, fx.Node, fx.Node] | None: - """Search graph for the GLU pattern `act(gate(x)) * up(x)`.""" - for mul in graph.nodes: - if ( - mul.op == "call_function" - and mul.target == operator.mul - and len(mul.args) == 2 - and all(isinstance(arg, fx.Node) for arg in mul.args) - ): - a, b = mul.args - if cls._is_act_of_gate(a, module) and is_linear(b, module): - act, gate, up = a, a.args[0], b - elif cls._is_act_of_gate(b, module) and is_linear(a, module): - act, gate, up = b, b.args[0], a - else: - continue - if ( - all(len(args) == 1 for args in (gate.args, up.args)) - and isinstance(x := gate.args[0], fx.Node) - and x is up.args[0] - ): - return act, gate, up, mul - return None - - @staticmethod - def _get_act_and_mul_name(act: nn.Module) -> str | None: - """Get the name of `act` if it has an `...AndMul` equivalent.""" - for name in CLS2ACT.get(type(act), []): - if name in ACT_AND_MUL_NAMES: - return name - # nn.GELU is not in ACT2CLS, but could be in model code - if type(act) is nn.GELU: - return "gelu_pytorch_tanh" if act.approximate == "tanh" else "gelu" - return None - - @classmethod - def _get_act_and_mul(cls, act: nn.Module) -> nn.Module: - """Get the `...AndMul` equivalent of a Transformers activation module.""" - if name := cls._get_act_and_mul_name(act): - return get_act_and_mul_fn(name) - raise ValueError(f"No AndMul equivalent for {type(act)}") - - @classmethod - def match(cls, graph: fx.Graph, module: nn.Module) -> "GLUFuser | None": - if (glu_nodes := cls._get_glu_nodes(graph, module)) is None: - return None - act_node, gate_node, up_node, mul_node = glu_nodes - - predicate = lambda n: is_linear(n, module) and peel(n.args[0]) is mul_node - down_node = find_node(graph, predicate) - return cls( - source_cls=type(module).__name__, - act_name=act_node.target, - linear_names=(gate_node.target, up_node.target), - down_name=down_node.target if down_node is not None else None, - ) - - def update_forward(self, module: nn.Module) -> None: - """Replace `act(gate(x)) * up(x)` with `act(gate_up(x))` in source.""" - funcdef, fn = recover_forward(type(module)) - act_call = single_self_call(funcdef, self.act_name) - calls = self._unguarded_calls(funcdef, (self.gate_name, self.up_name)) - gate_call, up_call = calls - if act_call.args[0] is not gate_call: - raise ValueError("activation does not directly wrap the gate") - if ast.dump(gate_call.args[0]) != ast.dump(up_call.args[0]): - raise ValueError("gate and up inputs are written differently") - muls = [ - node - for node in ast.walk(funcdef) - if isinstance(node, ast.BinOp) - and isinstance(node.op, ast.Mult) - and {id(node.left), id(node.right)} == {id(act_call), id(up_call)} - ] - if len(muls) != 1: - raise ValueError("no multiply of the activation and up projection") - - # act(gate(x)) * up(x) -> act(gate_up(x)) - assert isinstance(gate_call.func, ast.Attribute) - gate_call.func.attr = self.merged_name - replace_expr(funcdef, muls[0], act_call) - self.fused_forward = compile_forward(funcdef, fn) - - def validate(self, module: nn.Module, vllm_config: "VllmConfig") -> bool: - if not super().validate(module, vllm_config): - return False - # An AndMul kernel splits its input in half, so gate and up must match. - gate = module.get_submodule(self.gate_name) - up = module.get_submodule(self.up_name) - if gate.out_features != up.out_features: - logger.debug("%s and %s differ in size; skipping fusion", gate, up) - return False - act = module.get_submodule(self.act_name) - if self._get_act_and_mul_name(act) is None: - logger.debug("%s has no AndMul equivalent; skipping fusion", act) - return False - return True - - def update_attrs( - self, module: nn.Module, prefix: str, vllm_config: "VllmConfig" - ) -> None: - super().update_attrs(module, prefix, vllm_config) - act_fn = self._get_act_and_mul(module.get_submodule(self.act_name)) - setattr(module, self.act_name, act_fn) - # If there is a down projection, we know it must be rowwise. - if self.down_name is not None: - down_prefix = maybe_prefix(prefix, self.down_name) - down = module.get_submodule(self.down_name) - new_down = replace_linear_class( - down, "rowwise", vllm_config.quant_config, prefix=down_prefix - ) - setattr(module, self.down_name, new_down) - log_replacement(down_prefix, down, new_down) diff --git a/vllm/model_executor/models/transformers/fusers/merged_column.py b/vllm/model_executor/models/transformers/fusers/merged_column.py deleted file mode 100644 index 4a2b9c2b09..0000000000 --- a/vllm/model_executor/models/transformers/fusers/merged_column.py +++ /dev/null @@ -1,145 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Fuser for parallel linear projections.""" - -import ast -from dataclasses import dataclass -from typing import TYPE_CHECKING, ClassVar - -from torch import fx, nn - -from vllm.logger import init_logger -from vllm.model_executor.layers.linear import MergedColumnParallelLinear -from vllm.model_executor.models.transformers.fusers.base import StackedFuser -from vllm.model_executor.models.transformers.fx_utils import ( - block_chain, - compile_forward, - is_linear, - recover_forward, -) -from vllm.model_executor.models.utils import ShardId, maybe_prefix - -if TYPE_CHECKING: - from vllm.config import VllmConfig - -logger = init_logger(__name__) - - -@dataclass -class MergedColumnParallelFuser(StackedFuser): - """Fuser for merging column-parallel linear projections.""" - - linear_names: tuple[str, ...] - merged_cls_name: ClassVar[str] = "MergedColumnParallelLinear" - - @property - def merged_name(self) -> str: - """Programmatic name for the merged projection, based on the original names.""" - # len is used to disambiguate names like `a_b` + `c` vs `a` + `b_c` - parts = "_".join(f"{len(name)}_{name}" for name in self.linear_names) - return f"merged_proj_{parts}" - - @property - def shards(self) -> list[tuple[str, ShardId]]: - return [(name, index) for index, name in enumerate(self.linear_names)] - - @staticmethod - def _names(group: list[fx.Node]) -> tuple[str, ...] | None: - names = tuple(str(node.target) for node in group) - if len(set(names)) != len(names) or any("." in name for name in names): - return None - return names - - @classmethod - def match( - cls, graph: fx.Graph, module: nn.Module - ) -> "MergedColumnParallelFuser | None": - """Fuse the module's sibling linears when there is only one such group.""" - by_input: dict[fx.Node, list[fx.Node]] = {} - for node in graph.nodes: - if ( - is_linear(node, module) - and len(node.args) == 1 - and not node.kwargs - and isinstance(node.args[0], fx.Node) - # Like QKVFuser/PackedQKVFuser: this fuser has no head or - # TP-replication awareness, so it must not absorb a QKV-shaped - # pattern (e.g. behind a norm) a more specific fuser declined. - and node.args[0].op == "placeholder" - ): - by_input.setdefault(node.args[0], []).append(node) - # A group whose members are not distinct direct children is dropped: - # the source rewrite addresses each projection as `self.` once. - groups = [ - (group, names) - for group in by_input.values() - if len(group) >= 2 and (names := cls._names(group)) is not None - ] - if len(groups) > 1: - logger.debug( - "%s has %d fusable sibling-linear groups; skipping fusion " - "since which one to merge is ambiguous", - type(module).__name__, - len(groups), - ) - if len(groups) != 1: - return None - _, names = groups[0] - fuser = cls(source_cls=type(module).__name__, linear_names=names) - return None if hasattr(module, fuser.merged_name) else fuser - - def update_forward(self, module: nn.Module) -> None: - """Replace the parallel calls with one merged call and split.""" - funcdef, fn = recover_forward(type(module)) - calls = self._unguarded_calls(funcdef, self.linear_names) - if len(set(ast.dump(call.args[0]) for call in calls)) != 1: - raise ValueError("parallel linears read different inputs") - chains = [block_chain(funcdef.body, call) for call in calls] - if any(not chain for chain in chains): - raise ValueError("parallel linear calls not found in the function body") - blocks = [chain[-1] for chain in chains] - if len(set(id(block) for block, _ in blocks)) != 1: - raise ValueError("parallel linear calls are in different blocks") - - block = blocks[0][0] - indices = [index for _, index in blocks] - index = min(indices) - self._check_input_stable(funcdef, module, calls, block, indices) - - # l1(x), l2(x), ... -> merged(x).split(merged.output_sizes / merged.tp_size, -1) - self._splice_merged_split(funcdef, calls, block, index) - self.fused_forward = compile_forward(funcdef, fn) - - def validate(self, module: nn.Module, vllm_config: "VllmConfig") -> bool: - """Check that the parallel linears are compatible for merging.""" - linear_layers = [module.get_submodule(name) for name in self.linear_names] - tp_size = vllm_config.parallel_config.tensor_parallel_size - return ( - len(linear_layers) >= 2 - and len(set(linear.in_features for linear in linear_layers)) == 1 - and len(set(linear.bias is None for linear in linear_layers)) == 1 - and all(linear.out_features % tp_size == 0 for linear in linear_layers) - ) - - def update_attrs( - self, module: nn.Module, prefix: str, vllm_config: "VllmConfig" - ) -> None: - """Replace the module's parallel linears with one merged projection.""" - linear_modules = [module.get_submodule(name) for name in self.linear_names] - merged = MergedColumnParallelLinear( - input_size=linear_modules[0].in_features, - output_sizes=[linear.out_features for linear in linear_modules], - bias=linear_modules[0].bias is not None, - quant_config=vllm_config.quant_config, - prefix=maybe_prefix(prefix, self.merged_name), - return_bias=False, - ) - setattr(module, self.merged_name, merged) - for name in self.linear_names: - delattr(module, name) - logger.debug( - "%s -> %s: %s", - ", ".join(f"{n}: {m}" for n, m in zip(self.linear_names, linear_modules)), - self.merged_name, - merged, - ) diff --git a/vllm/model_executor/models/transformers/fusers/mla.py b/vllm/model_executor/models/transformers/fusers/mla.py deleted file mode 100644 index 96a4a0b87b..0000000000 --- a/vllm/model_executor/models/transformers/fusers/mla.py +++ /dev/null @@ -1,303 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""MLA fuser: adapt a Transformers MLA attention module for vLLM's MLA layer.""" - -import ast -import inspect -import textwrap -from dataclasses import dataclass -from typing import TYPE_CHECKING, ClassVar - -from torch import fx, nn - -from vllm.logger import init_logger -from vllm.model_executor.layers.linear import MergedColumnParallelLinear -from vllm.model_executor.models.transformers.fusers.base import StackedFuser -from vllm.model_executor.models.transformers.fusers.rms_norm import RMSNormFuser -from vllm.model_executor.models.transformers.fx_utils import ( - compile_forward, - downstream_linear, - is_linear, - recover_forward, - replace_expr, - returned_linear, - upstream_linear, -) -from vllm.model_executor.models.transformers.utils import ( - Style, - log_replacement, - replace_linear_class, -) -from vllm.model_executor.models.utils import ShardId, maybe_prefix - -if TYPE_CHECKING: - from vllm.config import VllmConfig - -logger = init_logger(__name__) - - -def _consumes_placeholder(node: fx.Node) -> bool: - """Whether `node` is a linear applied directly to a `forward` input.""" - return ( - len(node.args) == 1 - and isinstance(node.args[0], fx.Node) - and node.args[0].op == "placeholder" - ) - - -def _norm_size(norm: nn.Module) -> int: - weight = getattr(norm, "weight", None) - return weight.numel() if weight is not None else -1 - - -def _is_rms_norm(module: nn.Module) -> bool: - """Whether `module` computes an RMSNorm, verified by `RMSNormFuser`'s matcher.""" - # Go via `get_fuser` because it caches - from vllm.model_executor.models.transformers.fuser import get_fuser - - return get_fuser(module, RMSNormFuser) is not None - - -def _top_level_index(funcdef: ast.FunctionDef, node: ast.AST) -> int: - """Index in `funcdef.body` of the top-level statement containing `node`.""" - for index, stmt in enumerate(funcdef.body): - if any(child is node for child in ast.walk(stmt)): - return index - raise ValueError("node is not in the function body") - - -def _single_expand_call( - funcdef: ast.FunctionDef, module: nn.Module, kv_b_proj_name: str -) -> ast.Call: - """The KV expansion call `self.(kv_c_normed, k_pe)`: the unique - two-argument call to a method of `module` with the expansion's signature. - - The expansion is `kv_c_normed, k_pe -> key, value`: two inputs, every - `return` a pair, and it applies `kv_b_proj` -- the projection `MLAAttention` - owns after the bypass, absorbed into the attention computation. - """ - - def is_expansion_method(name: str) -> bool: - method = getattr(type(module), name, None) - if method is None: - return False - code = getattr(inspect.unwrap(method), "__code__", None) - if ( - code is None - or code.co_argcount != 3 # self, kv_c_normed, k_pe - or kv_b_proj_name not in code.co_names - ): - return False - try: - tree = ast.parse(textwrap.dedent(inspect.getsource(method))) - except (OSError, TypeError): - return False - returns = [node for node in ast.walk(tree) if isinstance(node, ast.Return)] - return bool(returns) and all( - isinstance(ret.value, ast.Tuple) and len(ret.value.elts) == 2 - for ret in returns - ) - - calls = [ - node - for node in ast.walk(funcdef) - if isinstance(node, ast.Call) - and isinstance(node.func, ast.Attribute) - and isinstance(node.func.value, ast.Name) - and node.func.value.id == "self" - and len(node.args) == 2 - and not node.keywords - and is_expansion_method(node.func.attr) - ] - if len(calls) != 1: - raise ValueError("expected exactly one KV expansion method call") - return calls[0] - - -@dataclass -class MLAFuser(StackedFuser): - """Fuser for the MLA attention pattern.""" - - q_proj_name: str | None - q_a_proj_name: str | None - q_a_layernorm_name: str | None - q_b_proj_name: str | None - kv_a_proj_name: str - kv_a_layernorm_name: str - kv_b_proj_name: str - o_proj_name: str | None - merged_name: ClassVar[str] = "fused_qkv_a_proj" - merged_cls_name: ClassVar[str] = "MergedColumnParallelLinear" - - def info(self, name: str) -> str: - info_str = ( - f"Fused: {name} ({self.source_cls}) -> MLAAttention (attention interface)" - ) - if self.q_a_proj_name is not None: - info_str += "; " + super().info(name).removeprefix("Fused: ") - return info_str - - @property - def shards(self) -> list[tuple[str, ShardId]]: - """`q_a_proj` and `kv_a_proj_with_mqa` stack into one down-projection.""" - if self.q_a_proj_name is None: - return [] - return [(self.q_a_proj_name, 0), (self.kv_a_proj_name, 1)] - - @property - def packed_modules_mapping(self) -> dict[str, list[str]]: - if self.q_a_proj_name is None: - return {} - return super().packed_modules_mapping - - @classmethod - def match(cls, graph: fx.Graph, module: nn.Module) -> "MLAFuser | None": - # Find all `rms_norm(linear(placeholder))` chains. - chains = [] - for node in graph.nodes: - if node.op != "call_module" or is_linear(node, module) or not node.args: - continue - source = upstream_linear(node.args[0], module) - if source is None or not _consumes_placeholder(source): - continue - if not _is_rms_norm(module.get_submodule(node.target)): - continue - chains.append((source, node)) - - # Tell the chains apart by width. - def is_kv_a(chain) -> bool: - source, rms_norm = chain - source_mod = module.get_submodule(source.target) - rms_norm_mod = module.get_submodule(rms_norm.target) - return source_mod.out_features != _norm_size(rms_norm_mod) - - kv_a_chains = [chain for chain in chains if is_kv_a(chain)] - q_a_chains = [chain for chain in chains if not is_kv_a(chain)] - # Exactly one KV chain is MLA's signature. The query chain is optional. - if len(kv_a_chains) != 1 or len(q_a_chains) > 1: - return None - kv_a_proj, kv_a_layernorm = kv_a_chains[0] - - # Linear children claimed for a role so far; the rest resolve by elimination. - claimed_linears = {kv_a_proj.target} - q_proj_name = q_a_proj = q_a_layernorm = q_b_proj = None - if q_a_chains: - # Find `q_b_proj(q_a_layernorm(q_a_proj(placeholder)))`. - q_a_proj, q_a_layernorm = q_a_chains[0] - q_b_proj = downstream_linear(q_a_layernorm, module) - if q_b_proj is None: - return None - claimed_linears |= {q_a_proj.target, q_b_proj.target} - else: - # Find `q_proj(placeholder)`. - placeholder_linears = { - node.target - for node in graph.nodes - if is_linear(node, module) and _consumes_placeholder(node) - } - if len(q_proj_candidates := placeholder_linears - claimed_linears) != 1: - return None - q_proj_name = next(iter(q_proj_candidates)) - claimed_linears.add(q_proj_name) - - # Find `kv_b_proj(kv_a_layernorm(...))`. - kv_b_proj = downstream_linear(kv_a_layernorm, module) - if kv_b_proj is None or kv_b_proj.target in claimed_linears: - return None - claimed_linears.add(kv_b_proj.target) - - # Find `o_proj` if it is returned by the forward graph. - o_proj_name = returned_linear(graph, module) - if o_proj_name in claimed_linears: - o_proj_name = None - - return cls( - source_cls=type(module).__name__, - q_proj_name=q_proj_name, - q_a_proj_name=q_a_proj.target if q_a_proj else None, - q_a_layernorm_name=q_a_layernorm.target if q_a_layernorm else None, - q_b_proj_name=q_b_proj.target if q_b_proj else None, - kv_a_proj_name=kv_a_proj.target, - kv_a_layernorm_name=kv_a_layernorm.target, - kv_b_proj_name=kv_b_proj.target, - o_proj_name=o_proj_name, - ) - - def validate(self, module: nn.Module, vllm_config: "VllmConfig") -> bool: - return vllm_config.model_config.use_mla - - def update_forward(self, module: nn.Module) -> None: - """Merge `q_a_proj` and `kv_a_proj` into one fused down proj then split. - Bypass the KV expansion method so the compressed latent reaches the `vllm_mla` - attention interface unexpanded.""" - funcdef, fn = recover_forward(type(module)) - if (q_a_proj_name := self.q_a_proj_name) is not None: - # q_a_proj is usually inside the `else` of `if self.q_lora_rank is None`. - # The fused call is inserted at the top-level statement preceding both. - names = [q_a_proj_name, self.kv_a_proj_name] - calls = self._unguarded_calls(funcdef, names) - if ast.dump(calls[0].args[0]) != ast.dump(calls[1].args[0]): - raise ValueError("down-projections read different inputs") - indices = [_top_level_index(funcdef, call) for call in calls] - self._check_input_stable(funcdef, module, calls, funcdef.body, indices) - self._splice_merged_split(funcdef, calls, funcdef.body, min(indices)) - - # Transformers expands the latent into full key/value in a dedicated method. - # `MLAAttention` consumes the latent directly (absorbing `kv_b_proj`), - # so replace the expansion call with its own arguments so `kv_c_normed, k_pe` - # flow to the interface in place of `key, value`. - expand_call = _single_expand_call(funcdef, module, self.kv_b_proj_name) - replace_expr( - funcdef, expand_call, ast.Tuple(elts=list(expand_call.args), ctx=ast.Load()) - ) - self.fused_forward = compile_forward(funcdef, fn) - - def update_attrs(self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"): - quant_config = vllm_config.quant_config - - def replace_linear_by_name(name: str, style: Style): - linear = module.get_submodule(name) - replacement = replace_linear_class( - linear, style, quant_config, prefix=maybe_prefix(prefix, name) - ) - setattr(module, name, replacement) - log_replacement(maybe_prefix(prefix, name), linear, replacement) - - if (q_a_proj_name := self.q_a_proj_name) is not None: - # `match` sets the q-LoRA names together, or none of them - assert self.q_b_proj_name is not None - q_a = module.get_submodule(q_a_proj_name) - kv_a = module.get_submodule(self.kv_a_proj_name) - merged = MergedColumnParallelLinear( - input_size=q_a.in_features, - output_sizes=[q_a.out_features, kv_a.out_features], - bias=q_a.bias is not None, - quant_config=quant_config, - prefix=maybe_prefix(prefix, self.merged_name), - return_bias=False, - disable_tp=True, - ) - logger.debug( - "%s: %s, %s: %s -> %s: %s", - q_a_proj_name, - q_a, - self.kv_a_proj_name, - kv_a, - self.merged_name, - merged, - ) - setattr(module, self.merged_name, merged) - # The rewritten forward calls the merged projection instead. - delattr(module, q_a_proj_name) - delattr(module, self.kv_a_proj_name) - replace_linear_by_name(self.q_b_proj_name, "colwise") - else: - assert self.q_proj_name is not None - replace_linear_by_name(self.kv_a_proj_name, "replicate") - replace_linear_by_name(self.q_proj_name, "colwise") - - replace_linear_by_name(self.kv_b_proj_name, "colwise") - # MLAAttention calls kv_b_proj and expects vLLM's default return_bias=True - module.get_submodule(self.kv_b_proj_name).return_bias = True - if self.o_proj_name is not None: - replace_linear_by_name(self.o_proj_name, "rowwise") diff --git a/vllm/model_executor/models/transformers/fusers/moe.py b/vllm/model_executor/models/transformers/fusers/moe.py deleted file mode 100644 index 46a1ce33bc..0000000000 --- a/vllm/model_executor/models/transformers/fusers/moe.py +++ /dev/null @@ -1,439 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""MoE fuser: run an HF MoE block's gate, experts and shared experts in `FusedMoE`.""" - -import ast -import functools -import inspect -import textwrap -import types -from collections.abc import Iterable, Iterator -from dataclasses import dataclass - -import torch -from torch import fx, nn -from typing_extensions import TypeIs - -from vllm.distributed import tensor_model_parallel_all_gather -from vllm.logger import init_logger -from vllm.model_executor.layers.fused_moe import GateLinear -from vllm.model_executor.layers.linear import ReplicatedLinear -from vllm.model_executor.models.transformers.fx_utils import ( - find_node, - is_complete, - is_op, - output_value, - peel, - trace, -) -from vllm.model_executor.models.transformers.utils import named_state -from vllm.model_executor.models.utils import maybe_prefix, sequence_parallel_chunk - -logger = init_logger(__name__) - - -def _own_returns(node: ast.AST) -> Iterator[ast.Return]: - """`return` statements in `node`'s own scope, not in nested functions.""" - stack = list(ast.iter_child_nodes(node)) - while stack: - child = stack.pop() - if isinstance(child, ast.Return): - yield child - elif not isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef, ast.Lambda)): - stack.extend(ast.iter_child_nodes(child)) - - -def _returns_tuple(cls: type[nn.Module]) -> bool: - """Does `cls.forward()` return a tuple?""" - try: - source = textwrap.dedent(inspect.getsource(inspect.unwrap(cls.forward))) - forward = ast.parse(source).body[0] - except (OSError, SyntaxError, TypeError, IndexError): - return True - # Names bound to a tuple literal, e.g. `out = hidden, logits` then `return out`. - tuple_names = { - target.id - for node in ast.walk(forward) - if isinstance(node, ast.Assign) and isinstance(node.value, ast.Tuple) - for target in node.targets - if isinstance(target, ast.Name) - } - - def yields_tuple(value: ast.expr | None) -> bool: - if isinstance(value, ast.Tuple): - return True - if isinstance(value, ast.Name): - return value.id in tuple_names - if isinstance(value, ast.IfExp): - return yields_tuple(value.body) or yields_tuple(value.orelse) - return False - - return any(yields_tuple(ret.value) for ret in _own_returns(forward)) - - -def _is_scalar_gate(module: nn.Module) -> bool: - """A linear projecting to a single logit (the shared-expert sigmoid gate).""" - weight = getattr(module, "weight", None) - return ( - isinstance(module, nn.Linear) - and weight is not None - and weight.ndim == 2 - and weight.shape[0] == 1 - ) - - -def _forced_dtype(nodes: Iterable[fx.Node]) -> torch.dtype | None: - """The floating dtype `nodes` cast to, if any. - - Computations that must run in higher precision say so in their forward (e.g. - `hidden_states.type(torch.float32)`), so the dtype is readable from the graph - even when the config does not name it.""" - for node in nodes: - if node.op not in ("call_method", "call_function"): - continue - name = str(node.target).rsplit(".", 1)[-1] - if name == "float": - return torch.float32 - if name in ("to", "type"): - for arg in (*node.args[1:], *node.kwargs.values()): - if isinstance(arg, torch.dtype) and arg.is_floating_point: - return arg - return None - - -def _reaches(node: fx.Node, key: str) -> set[fx.Node]: - """Returns the set of nodes reachable from `node` by following `key` edges.""" - seen: set[fx.Node] = set() - stack = [node] - while stack: - n = stack.pop() - if n in seen: - continue - seen.add(n) - stack.extend(getattr(n, key)) - return seen - - -def _renormalizes(topk: fx.Node) -> bool: - """Are the top-k weights divided by their own sum? - - Every HF router spells renormalization as a division by a sum over the - selection, optionally offset by an epsilon.""" - downstream = _reaches(topk, "users") - for node in downstream: - if not (is_op(node, "div") or is_op(node, "truediv")) or len(node.args) < 2: - continue - denominator = node.args[1] - if not isinstance(denominator, fx.Node): - continue - cone = _reaches(denominator, "all_input_nodes") - if any(is_op(n, "sum") and n in downstream for n in cone): - return True - return False - - -def _unwrap(node: object, moe_block: nn.Module) -> object: - """Strip reshapes, dtype casts and `nn.Identity` calls.""" - while True: - node = peel(node) - if ( - is_op(node, "reshape") - or is_op(node, "view") - or ( - isinstance(node, fx.Node) - and node.op == "call_module" - and isinstance(moe_block.get_submodule(node.target), nn.Identity) - ) - ): - node = node.args[0] - else: - return node - - -def _reads_input(node: object, x: fx.Node, moe_block: nn.Module) -> TypeIs[fx.Node]: - """Is `node` a child module of `moe_block` called on just `x`?""" - return ( - isinstance(node, fx.Node) - and node.op == "call_module" - and len(node.args) == 1 - and not node.kwargs - and _unwrap(node.args[0], moe_block) is x - ) - - -class SharedExpertMLP(nn.Module): - """Wraps an HF shared expert, applying the output gating it is paired with.""" - - def __init__(self, shared_experts: nn.Module, gate: nn.Module | None = None): - super().__init__() - self.shared_experts = shared_experts - self.gate = gate - - def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: - out = self.shared_experts(hidden_states) - if self.gate is not None: - out = torch.sigmoid(self.gate(hidden_states)[0]) * out - return out - - -def _moe_block_forward(self: nn.Module, hidden_states: torch.Tensor) -> torch.Tensor: - """Standard MoE block forward. - - Routing and any shared experts are handled inside `self.experts: MoERunner`.""" - orig_shape = hidden_states.shape - hidden_states = hidden_states.reshape(-1, orig_shape[-1]) - num_tokens = hidden_states.shape[0] - is_sequence_parallel = self.experts.moe_config.is_sequence_parallel - if is_sequence_parallel: - hidden_states = sequence_parallel_chunk(hidden_states) - out = self.experts(hidden_states, router_logits=hidden_states) - if is_sequence_parallel: - out = tensor_model_parallel_all_gather(out, 0)[:num_tokens] - return out.reshape(orig_shape) - - -@dataclass -class MoEBlockFuser: - """Fuser for MoE block `experts`, `gate` and `shared_experts` (optional).""" - - gate_name: str - scoring_func: str | None - """`None` if routing stays in the Transformers gate.""" - shared_name: str | None - shared_gate_name: str | None - router_dtype: torch.dtype | None = None - renormalize: bool | None = None - - @staticmethod - def _match_router( - gate: nn.Module, - ) -> tuple[str, torch.dtype | None, bool | None] | None: - """Matches `topk(score(linear(x)))` or `score(topk(linear(x)))`, `score` being - `softmax`/`sigmoid`. - - Returns the scoring function, the dtype the router computes in, and - whether the top-k weights are renormalized (`None` if the trace was cut - short before it could say).""" - state = {name for name, _ in named_state(gate)} - if "weight" not in state or state - {"weight", "e_score_correction_bias"}: - return None - graph = trace(gate) - if graph is None: - return None - # The routing top-k is the last one; any earlier one scores expert groups. - topks = [node for node in graph.nodes if is_op(node, "topk")] - if not topks: - return None - topk = topks[-1] - is_scorer = lambda n: is_op(n, "softmax") or is_op(n, "sigmoid") - post_topk = False - # Find all scoring functions that reach the top-k node. - scorers = [n for n in _reaches(topk, "all_input_nodes") if is_scorer(n)] - if not scorers: - # Find all scoring functions that are users of the top-k node. - scorers = [n for n in _reaches(topk, "users") if is_scorer(n)] - post_topk = True - if len(scorers) != 1: - return None - scorer = scorers[0] - logits_cone = _reaches(scorer, "all_input_nodes") - if not any(is_op(n, "linear") for n in logits_cone): - return None - is_softmax = is_op(scorer, "softmax") - # vLLM always scores every expert, so matching a router that softmaxes - # the selection alone takes renormalizing. Otherwise look for the - # division, reading its absence as "no" only from a complete trace. - renormalize: bool | None - if (post_topk and is_softmax) or _renormalizes(topk): - renormalize = True - else: - renormalize = False if is_complete(graph) else None - scoring_func = "softmax" if is_softmax else "sigmoid" - return scoring_func, _forced_dtype(logits_cone), renormalize - - @staticmethod - def _match_experts( - graph: fx.Graph, experts_name: str, moe_block: nn.Module - ) -> tuple[fx.Node, fx.Node, fx.Node] | None: - """Matches `experts(x, ids, weights)` with `_, weights, ids = gate(x)`, `x` - being the block's input. - - Returns the experts call, `x` and the gate call.""" - experts = find_node( - graph, lambda n: n.op == "call_module" and n.target == experts_name - ) - if experts is None: - return None - # Bind by signature so keyword calls match too. Fail closed on any other - # argument: the rewritten forward would drop it. - forward = inspect.signature(moe_block.get_submodule(experts_name).forward) - try: - bound = forward.bind(*experts.args, **experts.kwargs) - except TypeError: - return None - if len(bound.arguments) != 3: - return None - x, ids, weights = (_unwrap(a, moe_block) for a in bound.arguments.values()) - if not (isinstance(x, fx.Node) and x.op == "placeholder"): - return None - if not (is_op(ids, "getitem") and is_op(weights, "getitem")): - return None - gate = ids.args[0] - if not ( - _reads_input(gate, x, moe_block) - and ids.args == (gate, 2) - and weights.args == (gate, 1) - ): - return None - return experts, x, gate - - @staticmethod - def _match_shared_experts( - node: object, x: fx.Node, moe_block: nn.Module - ) -> tuple[str, str | None] | None: - """Matches `shared(x)` or `sigmoid(shared_gate(x)) * shared(x)`. - - Returns the shared expert's name and its optional gate's name.""" - if _reads_input(node, x, moe_block): - return node.target, None - if not (is_op(node, "mul") and len(node.args) == 2): - return None - for gated, shared in (node.args, node.args[::-1]): - gated, shared = peel(gated), peel(shared) - if not (is_op(gated, "sigmoid") and _reads_input(shared, x, moe_block)): - continue - gate = peel(gated.args[0]) - if _reads_input(gate, x, moe_block) and _is_scalar_gate( - moe_block.get_submodule(gate.target) - ): - return shared.target, gate.target - return None - - @classmethod - def match( - cls, - moe_block: nn.Module, - experts_name: str, - divides_shared_output: bool = False, - ) -> "MoEBlockFuser | None": - """Matches a block returning `experts(x, ids, weights) [* c] [+ shared(x)]` - with `_, weights, ids = gate(x)`, `x` being the block's input, up to - reshapes, casts and `nn.Identity`. - - Args: - moe_block: The MoE block. - experts_name: The name of the experts in `moe_block`. - divides_shared_output: Whether vLLM routing would divide the shared - expert's output by `routed_scaling_factor` (fp16 overflow - protection), which Transformers decoder layers do not undo. - - Returns: - A fuser if more than the experts can run in vLLM, else `None`. - - """ - # Standard MoE block returns a single tensor. - if _returns_tuple(type(moe_block)): - return None - if (graph := trace(moe_block)) is None: - return None - if (matched := cls._match_experts(graph, experts_name, moe_block)) is None: - return None - experts, x, gate = matched - unwrap = functools.partial(_unwrap, moe_block=moe_block) - gate_name = gate.target - router = cls._match_router(getattr(moe_block, gate_name)) - scoring_func, router_dtype, renormalize = router or (None, None, None) - - def is_routed(n: object) -> bool: - return n is experts or ( - is_op(n, "mul") - and not isinstance(n.args[1], fx.Node) - and unwrap(n.args[0]) is experts - ) - - # Output: the experts' output, plus the shared expert's if any. - routed = out = unwrap(output_value(graph)) - shared = None - if not is_routed(out): - if not (is_op(out, "add") and len(out.args) == 2): - return None - routed, shared = map(unwrap, out.args) - if is_routed(shared): - routed, shared = shared, routed - if not is_routed(routed): - return None - shared_name = shared_gate_name = None - if shared is not None: - if (match := cls._match_shared_experts(shared, x, moe_block)) is None: - return None - shared_name, shared_gate_name = match - # Fail closed: `rewrite_forward` runs only the experts and the detected - # shared expert, so any other stateful child would be dropped. - accounted = {experts_name, gate_name, shared_name, shared_gate_name} - for name, child in moe_block.named_children(): - if name not in accounted and next(named_state(child), None) is not None: - return None - if scoring_func is not None and shared_name and divides_shared_output: - logger.warning_once( - "%s could be fused but routing it in vLLM would apply " - "`routed_scaling_factor` by dividing the shared expert output, " - "which only fp16 overflow protection expects the decoder layer to " - "compensate for. Falling back to routing in Transformers; run in " - "bfloat16 to fuse it.", - type(moe_block).__name__, - ) - scoring_func = router_dtype = renormalize = None - # Without vLLM routing, only a shared expert can move to vLLM, and only - # if the block does not scale the experts' output (vLLM routing does). - if scoring_func is None and (shared_name is None or routed is not experts): - return None - return cls( - gate_name, - scoring_func, - shared_name, - shared_gate_name, - router_dtype, - renormalize, - ) - - def gate( - self, moe_block: nn.Module, prefix: str, out_dtype: torch.dtype | None = None - ) -> GateLinear: - """Rebuild the HF gate as a `GateLinear` for vLLM's fused MoE.""" - hf_gate = getattr(moe_block, self.gate_name) - num_experts, hidden_size = hf_gate.weight.shape - gate = GateLinear( - hidden_size, - num_experts, - bias=False, - out_dtype=out_dtype or self.router_dtype, - prefix=maybe_prefix(prefix, self.gate_name), - ) - if (bias := getattr(hf_gate, "e_score_correction_bias", None)) is not None: - gate.register_buffer("e_score_correction_bias", bias) - setattr(moe_block, self.gate_name, gate) - return gate - - def shared_experts( - self, moe_block: nn.Module, prefix: str - ) -> SharedExpertMLP | None: - """Build the HF shared expert (and its optional gate) - as a `SharedExpertMLP` for vLLM's fused MoE.""" - if self.shared_name is None: - return None - shared_experts = getattr(moe_block, self.shared_name) - gate = None - if self.shared_gate_name is not None: - hf_gate = getattr(moe_block, self.shared_gate_name) - gate = ReplicatedLinear( - hf_gate.in_features, - hf_gate.out_features, - bias=hf_gate.bias is not None, - prefix=maybe_prefix(prefix, self.shared_gate_name), - ) - setattr(moe_block, self.shared_gate_name, gate) - return SharedExpertMLP(shared_experts, gate) - - def rewrite_forward(self, moe_block: nn.Module) -> None: - """Rewrite `moe_block.forward` to route through vLLM's fused MoE.""" - moe_block.forward = types.MethodType(_moe_block_forward, moe_block) diff --git a/vllm/model_executor/models/transformers/fusers/packed_qkv.py b/vllm/model_executor/models/transformers/fusers/packed_qkv.py deleted file mode 100644 index 8b1b30b030..0000000000 --- a/vllm/model_executor/models/transformers/fusers/packed_qkv.py +++ /dev/null @@ -1,164 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Packed-QKV fuser: `c_attn(x).split((q, kv, kv))` -> a `QKVParallelLinear`.""" - -import ast -from dataclasses import dataclass -from typing import TYPE_CHECKING - -from torch import fx, nn - -from vllm.logger import init_logger -from vllm.model_executor.layers.linear import QKVParallelLinear -from vllm.model_executor.models.transformers.fusers.base import ( - RewriteFuser, - fused_head_size, - local_output_sizes, -) -from vllm.model_executor.models.transformers.fx_utils import ( - compile_forward, - is_method, - recover_forward, - returned_linear, - upstream_linear, -) -from vllm.model_executor.models.transformers.utils import ( - log_replacement, - replace_linear_class, -) -from vllm.model_executor.models.utils import maybe_prefix - -if TYPE_CHECKING: - from vllm.config import VllmConfig - -logger = init_logger(__name__) - - -@dataclass -class PackedQKVFuser(RewriteFuser): - """Fuser for attention with q, k and v packed into one projection.""" - - qkv_name: str - o_name: str | None - q_size: int - kv_size: int - - def info(self, name: str) -> str: - return ( - f"Fused: {self.qkv_name} ({name}: {self.source_cls}) -> QKVParallelLinear" - ) - - @staticmethod - def _packed_sizes(node: fx.Node) -> tuple[int, int] | None: - """`(q, kv)` from a `split((q, kv, kv), ...)` call, if it is one.""" - if not is_method(node, "split") or len(node.args) < 2: - return None - sizes = node.args[1] - if not isinstance(sizes, (tuple, list)) or len(sizes) != 3: - return None - if not all(isinstance(size, int) for size in sizes): - return None - q_size, k_size, v_size = sizes - if k_size != v_size or q_size < k_size: - return None - return q_size, k_size - - @classmethod - def match(cls, graph: fx.Graph, module: nn.Module) -> "PackedQKVFuser | None": - for node in graph.nodes: - if (sizes := cls._packed_sizes(node)) is None: - continue - q_size, kv_size = sizes - qkv_node = upstream_linear(node.args[0], module) - if qkv_node is None: - continue - qkv_name = str(qkv_node.target) - # The split must consume the whole projection. - if module.get_submodule(qkv_name).out_features != q_size + 2 * kv_size: - continue - # o_proj produces the module's output and consumes the query width. - o_name = returned_linear(graph, module) - if o_name == qkv_name or ( - o_name is not None - and module.get_submodule(o_name).in_features != q_size - ): - o_name = None - return cls( - source_cls=type(module).__name__, - qkv_name=qkv_name, - o_name=o_name, - q_size=q_size, - kv_size=kv_size, - ) - return None - - def _split_call(self, funcdef: ast.FunctionDef) -> ast.Call: - """The unique `self.(...)....split((a, b, c), ...)` call.""" - calls = [ - node - for node in ast.walk(funcdef) - if isinstance(node, ast.Call) - and isinstance(node.func, ast.Attribute) - and node.func.attr == "split" - and node.args - and isinstance(node.args[0], (ast.Tuple, ast.List)) - and len(node.args[0].elts) == 3 - and any( - isinstance(inner, ast.Attribute) and inner.attr == self.qkv_name - for inner in ast.walk(node.func.value) - ) - ] - if len(calls) != 1: - raise ValueError(f"{self.qkv_name} has {len(calls)} three-way splits") - return calls[0] - - def update_forward(self, module: nn.Module) -> None: - """Rewrite the split sizes to the sharded projection's per-rank widths.""" - funcdef, fn = recover_forward(type(module)) - split = self._split_call(funcdef) - # (q, kv, kv) -> [s // qkv.tp_size for s in qkv.output_sizes] - sections = local_output_sizes(self.qkv_name) - split.args[0] = ast.parse(sections, mode="eval").body - self.fused_forward = compile_forward(funcdef, fn) - - def validate(self, module: nn.Module, vllm_config: "VllmConfig") -> bool: - """Shapes must be compatible with a head-sharded packed GEMM.""" - head_size = fused_head_size(module, vllm_config) - qkv = module.get_submodule(self.qkv_name) - compatible = ( - self.q_size % head_size == 0 - and self.kv_size % head_size == 0 - and qkv.out_features == self.q_size + 2 * self.kv_size - ) - if not compatible: - logger.debug("%s is not compatible with packed QKV fusion", type(module)) - return compatible - - def update_attrs( - self, module: nn.Module, prefix: str, vllm_config: "VllmConfig" - ) -> None: - quant_config = vllm_config.quant_config - head_size = fused_head_size(module, vllm_config) - qkv_prefix = maybe_prefix(prefix, self.qkv_name) - qkv = module.get_submodule(self.qkv_name) - merged = QKVParallelLinear( - hidden_size=qkv.in_features, - head_size=head_size, - total_num_heads=self.q_size // head_size, - total_num_kv_heads=self.kv_size // head_size, - bias=qkv.bias is not None, - quant_config=quant_config, - prefix=qkv_prefix, - return_bias=False, - ) - setattr(module, self.qkv_name, merged) - log_replacement(qkv_prefix, qkv, merged) - # If there is an output projection, we know it must be rowwise. - if self.o_name is not None: - o_prefix = maybe_prefix(prefix, self.o_name) - o_proj = module.get_submodule(self.o_name) - new_o = replace_linear_class( - o_proj, "rowwise", quant_config, prefix=o_prefix - ) - setattr(module, self.o_name, new_o) - log_replacement(o_prefix, o_proj, new_o) diff --git a/vllm/model_executor/models/transformers/fusers/qkv.py b/vllm/model_executor/models/transformers/fusers/qkv.py deleted file mode 100644 index 7b5b727d10..0000000000 --- a/vllm/model_executor/models/transformers/fusers/qkv.py +++ /dev/null @@ -1,209 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""QKV projection fuser: `q(x), k(x), v(x)` -> a fused qkv linear + split.""" - -import ast -from dataclasses import dataclass -from typing import TYPE_CHECKING, ClassVar - -from torch import fx, nn - -from vllm.logger import init_logger -from vllm.model_executor.layers.linear import QKVParallelLinear -from vllm.model_executor.models.transformers.fusers.base import ( - StackedFuser, - fused_head_size, -) -from vllm.model_executor.models.transformers.fx_utils import ( - block_chain, - compile_forward, - is_linear, - recover_forward, - returned_linear, -) -from vllm.model_executor.models.transformers.utils import ( - log_replacement, - replace_linear_class, -) -from vllm.model_executor.models.utils import ShardId, maybe_prefix - -if TYPE_CHECKING: - from vllm.config import VllmConfig - -logger = init_logger(__name__) - - -@dataclass -class QKVFuser(StackedFuser): - """Fuser for the attention QKV pattern `q(x), k(x), v(x)`.""" - - q_name: str - k_name: str - v_name: str - o_name: str | None - merged_name: ClassVar[str] = "qkv_proj" - merged_cls_name: ClassVar[str] = "QKVParallelLinear" - - @property - def shards(self) -> list[tuple[str, ShardId]]: - return [(self.q_name, "q"), (self.k_name, "k"), (self.v_name, "v")] - - @classmethod - def _get_qkv_nodes( - cls, graph: fx.Graph, module: nn.Module - ) -> tuple[fx.Node, fx.Node, fx.Node] | None: - """Search `graph` for the QKV pattern `q(x), k(x), v(x)`.""" - by_input: dict[fx.Node, list[fx.Node]] = {} - for node in graph.nodes: - if ( - is_linear(node, module) - and len(node.args) == 1 - and not node.kwargs - and isinstance(node.args[0], fx.Node) - and node.args[0].op == "placeholder" - ): - by_input.setdefault(node.args[0], []).append(node) - triples = [nodes for nodes in by_input.values() if len(nodes) == 3] - if len(triples) != 1: - return None - - q_node, k_node, v_node = nodes = triples[0] - outs = [module.get_submodule(node.target).out_features for node in nodes] - if len(set(outs)) == 2: - # q is identified as the larger projection (GQA) - (q_node,) = (n for n, out in zip(nodes, outs) if outs.count(out) == 1) - k_node, v_node = (n for n, out in zip(nodes, outs) if outs.count(out) == 2) - if module.get_submodule(q_node.target).out_features != max(outs): - return None - elif len(set(outs)) != 1: - return None - return q_node, k_node, v_node - - @classmethod - def match(cls, graph: fx.Graph, module: nn.Module) -> "QKVFuser | None": - if (qkv_nodes := cls._get_qkv_nodes(graph, module)) is None: - return None - q, k, v = qkv_nodes - names = dict(q_name=q.target, k_name=k.target, v_name=v.target) - # o_proj produces the module's output. - o_name = returned_linear(graph, module) - # o_proj must be compatible with the q/k/v projections. - if o_name in names.values() or ( - o_name is not None - and module.get_submodule(o_name).in_features - != module.get_submodule(q.target).out_features - ): - o_name = None - names["o_name"] = o_name - return cls(source_cls=type(module).__name__, **names) - - def update_forward(self, module: nn.Module) -> None: - """Replace `q(x), k(x), v(x)` with `qkv(x).split(sizes, -1)` in source. - - A projection may be guarded by an existence check -- a - `self. is (not) None` comparison (e.g. Gemma 4's `v_proj`) or a bare - truthiness test -- which is folded to its constant value (see - `bypass_existence_guard`). The calls may sit in different branches, so the - fused GEMM is inserted before the earliest of them, in the innermost block - that dominates all three. - """ - funcdef, fn = recover_forward(type(module)) - proj_names = (self.q_name, self.k_name, self.v_name) - calls = self._unguarded_calls(funcdef, proj_names) - arg_dumps = {ast.dump(call.args[0]) for call in calls} - if len(arg_dumps) != 1: - raise ValueError("projection inputs are written differently") - # The trace may be partial, so prove projection exclusivity in source: - # no other linear child may consume the same input (else the matched - # three may not be q, k and v) - other_linears = { - name - for name, child in module.named_children() - if isinstance(child, nn.Linear) - } - set(proj_names) - for node in ast.walk(funcdef): - if ( - isinstance(node, ast.Call) - and isinstance(node.func, ast.Attribute) - and node.func.attr in other_linears - and any(ast.dump(arg) in arg_dumps for arg in node.args) - ): - raise ValueError("another linear consumes the same input") - - # Insert the fused GEMM before the earliest call, in the innermost block - # common to all three (the calls may be split across branches). - chains = [block_chain(funcdef.body, call) for call in calls] - if any(not chain for chain in chains): - raise ValueError("projection calls not found in the function body") - depth = 0 - for level in zip(*chains): - if len({id(block) for block, _ in level}) != 1: - break - depth += 1 - block = chains[0][depth - 1][0] - indices = [chain[depth - 1][1] for chain in chains] - insert_index = min(indices) - self._check_input_stable(funcdef, module, calls, block, indices) - - # q(x), k(x), v(x) -> q, k, v = qkv(x).split(qkv.output_sizes / qkv.tp_size, -1) - self._splice_merged_split(funcdef, calls, block, insert_index) - self.fused_forward = compile_forward(funcdef, fn) - - def validate(self, module: nn.Module, vllm_config: "VllmConfig") -> bool: - """Shapes must be compatible for a single merged, head-sharded GEMM.""" - q = module.get_submodule(self.q_name) - k = module.get_submodule(self.k_name) - v = module.get_submodule(self.v_name) - head_size = fused_head_size(module, vllm_config) - compatible = ( - q.in_features == k.in_features == v.in_features - and len({proj.bias is None for proj in (q, k, v)}) == 1 - and k.out_features == v.out_features - and q.out_features % head_size == 0 - and k.out_features % head_size == 0 - ) - if not compatible: - logger.debug("%s is not compatible with QKV fusion", type(module)) - return compatible - - def update_attrs( - self, module: nn.Module, prefix: str, vllm_config: "VllmConfig" - ) -> None: - quant_config = vllm_config.quant_config - head_size = fused_head_size(module, vllm_config) - q = module.get_submodule(self.q_name) - k = module.get_submodule(self.k_name) - merged = QKVParallelLinear( - hidden_size=q.in_features, - head_size=head_size, - total_num_heads=q.out_features // head_size, - total_num_kv_heads=k.out_features // head_size, - bias=q.bias is not None, - quant_config=quant_config, - prefix=maybe_prefix(prefix, self.merged_name), - return_bias=False, - ) - logger.debug( - "%s: %s, %s: %s, %s: %s -> %s: %s", - self.q_name, - q, - self.k_name, - k, - self.v_name, - module.get_submodule(self.v_name), - self.merged_name, - merged, - ) - setattr(module, self.merged_name, merged) - # Drop the consumed submodules so their (meta) params are not expected. - for name in (self.q_name, self.k_name, self.v_name): - delattr(module, name) - # If there is an output projection, we know it must be rowwise. - if self.o_name is not None: - o_proj_prefix = maybe_prefix(prefix, self.o_name) - o_proj = module.get_submodule(self.o_name) - new_o = replace_linear_class( - o_proj, "rowwise", quant_config, prefix=o_proj_prefix - ) - setattr(module, self.o_name, new_o) - log_replacement(o_proj_prefix, o_proj, new_o) diff --git a/vllm/model_executor/models/transformers/fusers/rms_norm.py b/vllm/model_executor/models/transformers/fusers/rms_norm.py deleted file mode 100644 index c1c7ea76ab..0000000000 --- a/vllm/model_executor/models/transformers/fusers/rms_norm.py +++ /dev/null @@ -1,297 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""RMSNorm fuser: detect the norm structurally and swap in vLLM's fused RMSNorm.""" - -from dataclasses import dataclass -from typing import TYPE_CHECKING - -import torch -from torch import fx, nn - -from vllm.distributed import ( - get_tensor_model_parallel_rank, - get_tensor_model_parallel_world_size, - tensor_model_parallel_all_gather, -) -from vllm.distributed.parallel_state import model_parallel_is_initialized -from vllm.distributed.utils import split_tensor_along_last_dim -from vllm.logger import init_logger -from vllm.model_executor.models.transformers.fusers.base import BaseFuser -from vllm.model_executor.models.transformers.fx_utils import ( - find_node, - forward_input_count, - is_op, - output_value, - peel, - trace, -) -from vllm.model_executor.models.transformers.layers import ( - GemmaRMSNorm, - RMSNorm, -) - -if TYPE_CHECKING: - from vllm.config import VllmConfig - -logger = init_logger(__name__) - - -def _operand(node: fx.Node, index: int, name: str) -> object | None: - """Operand `index` of `node`, whether it was passed positionally or as `name`.""" - if len(node.args) > index: - return node.args[index] - return node.kwargs.get(name) - - -def _is_squared(node: object, x: fx.Node) -> bool: - """`x**2`, `x.square()` or `x * x`, through any dtype casts.""" - node = peel(node) - if is_op(node, "pow"): - return ( - peel(_operand(node, 0, "input")) is x and _operand(node, 1, "exponent") == 2 - ) - if is_op(node, "square"): - return peel(_operand(node, 0, "input")) is x - if is_op(node, "mul"): - return ( - peel(_operand(node, 0, "input")) is x - and peel(_operand(node, 1, "other")) is x - ) - return False - - -def _is_inverse_sqrt(node: object) -> bool: - """Detect `rsqrt(v)`, or the `pow(v, -0.5)` / `v ** -0.5` spelling of it.""" - if is_op(node, "rsqrt"): - return True - if is_op(node, "pow"): - return _operand(node, 1, "exponent") == -0.5 - return False - - -def _variance_eps(rsqrt: fx.Node, x: fx.Node) -> float | None: - """`eps` from `rsqrt(mean(x**2, -1) + eps)`, or `None` if not that shape.""" - add = peel(_operand(rsqrt, 0, "input")) - if not is_op(add, "add"): - return None - operands = [_operand(add, 0, "input"), _operand(add, 1, "other")] - consts = [a for a in operands if isinstance(a, (int, float))] - nodes = [a for a in operands if isinstance(a, fx.Node)] - if len(consts) != 1 or len(nodes) != 1: - return None - mean = peel(nodes[0]) - if not is_op(mean, "mean"): - return None - if not _is_squared(_operand(mean, 0, "input"), x): - return None - return float(consts[0]) - - -def _is_one_plus(node: object) -> bool: - """`1 + weight` in either operand order (marks a zero-centered weight).""" - node = peel(node) - if not is_op(node, "add"): - return False - return any( - isinstance(a, (int, float)) and a == 1 - for a in (_operand(node, 0, "input"), _operand(node, 1, "other")) - ) - - -def _has_trailing_compute(graph: fx.Graph, node: fx.Node) -> bool: - """Does the forward compute anything after `node` before returning?""" - value = output_value(graph) - return value is not None and peel(value) is not node - - -class TPAwareNormMixin(nn.Module): - """Mixin for RMSNorms that reconstructs a TP-sharded input before normalizing.""" - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - if model_parallel_is_initialized(): - self.tp_size = get_tensor_model_parallel_world_size() - self.tp_rank = get_tensor_model_parallel_rank() - else: - self.tp_size, self.tp_rank = 1, 0 - - def forward( - self, x: torch.Tensor, residual: torch.Tensor | None = None - ) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]: - if self.tp_size > 1 and x.shape[-1] < (full := self.weight.shape[0]): - if x.shape[-1] * self.tp_size != full: - raise ValueError( - f"Cannot gather norm of width {full}: a TP-sharded input of " - f"width {x.shape[-1]} does not tile it evenly across " - f"{self.tp_size} ranks (replicated or uneven sharding)." - ) - x = tensor_model_parallel_all_gather(x.contiguous()) - x = super().forward(x) - splits = split_tensor_along_last_dim(x, num_partitions=self.tp_size) - return splits[self.tp_rank] - return super().forward(x, residual) - - -class TPAwareRMSNorm(TPAwareNormMixin, RMSNorm): - """`RMSNorm` that reconstructs a TP-sharded input before normalizing.""" - - -class TPAwareGemmaRMSNorm(TPAwareNormMixin, GemmaRMSNorm): - """`GemmaRMSNorm` that reconstructs a TP-sharded input before normalizing.""" - - -@dataclass -class RMSNormFuser(BaseFuser): - """Fuser for RMSNorm patterns, including Gemma-style zero-centered weights.""" - - zero_centered: bool - """Gemma-style `(1 + weight)` scaling (weight initialised at zero).""" - source_cls: str - """Class name of the norm this was matched from (for logging).""" - eps_attr: str | None = None - """Attribute holding `eps`, read per instance in `fuse`.""" - eps: float | None = None - """`eps` itself, when it is not held in an attribute (see `_eps_source`).""" - - def info(self, name: str) -> str: - norm = "GemmaRMSNorm" if self.zero_centered else "RMSNorm" - return f"Fused: {name} ({self.source_cls}) -> {norm} (CustomOp)" - - @classmethod - def match(cls, graph: fx.Graph, module: nn.Module) -> "RMSNormFuser | None": - """Match a graph to the RMSNorm pattern, returning a fuser if found.""" - if forward_input_count(type(module)) != 1: - return None - x = find_node(graph, lambda n: n.op == "placeholder") - if x is None: - return None - # Handle native torch `rms_norm` op. - rms_norm = find_node(graph, lambda n: is_op(n, "rms_norm")) - if rms_norm is not None and peel(_operand(rms_norm, 0, "input")) is x: - if _has_trailing_compute(graph, rms_norm): - return None - eps_attr, eps = cls._eps_source(graph, module) - return cls( - zero_centered=False, - source_cls=type(module).__name__, - eps_attr=eps_attr, - eps=eps, - ) - # Handle explicit `x * rsqrt(mean(x**2, -1) + eps)` pattern. - # The rsqrt over the mean-square variance is the spine of the norm. - rsqrt = None - for node in graph.nodes: - if _is_inverse_sqrt(node) and _variance_eps(node, x) is not None: - rsqrt = node - break - if rsqrt is None: - return None - # The `x * rsqrt(...)` normalize multiply. - normalize = find_node( - graph, - lambda n: is_op(n, "mul") - and rsqrt in map(peel, (_operand(n, 0, "input"), _operand(n, 1, "other"))), - ) - if normalize is None: - return None - # An optional later `weight * normalized` (or `(1 + weight) * normalized`). - tail, zero_centered = normalize, False - for node in graph.nodes: - if not is_op(node, "mul") or node is normalize: - continue - operands = [ - peel(a) - for a in (_operand(node, 0, "input"), _operand(node, 1, "other")) - if isinstance(a, fx.Node) - ] - if len(operands) == 2 and normalize in operands: - weight = next(o for o in operands if o is not normalize) - tail, zero_centered = node, _is_one_plus(weight) - break - # The norm must be the last compute in forward, or it is not a pure norm. - if _has_trailing_compute(graph, tail): - return None - eps_attr, eps = cls._eps_source(graph, module) - return cls( - zero_centered=zero_centered, - source_cls=type(module).__name__, - eps_attr=eps_attr, - eps=eps, - ) - - @classmethod - def _eps_source( - cls, graph: fx.Graph, module: nn.Module - ) -> tuple[str | None, float | None]: - """Where `fuse` should read `eps` from, resolved once per class.""" - eps = cls._eps_from_graph(graph) - if eps is None: - return None, None - # Whatever supplied the constant must still equal it. - candidates = { - name: value - for name, value in vars(module).items() - if isinstance(value, float) and value == eps - } - # Use unique markers and retrace to verify exactly which attribute is eps. - markers = {float(-index - 1): name for index, name in enumerate(candidates)} - marked = None - if markers: - try: - for marker, name in markers.items(): - setattr(module, name, marker) - if (remarked := trace(module)) is not None: - marked = cls._eps_from_graph(remarked) - finally: - for name, value in candidates.items(): - setattr(module, name, value) - if marked is not None and (name := markers.get(marked)) is not None: - return name, None - logger.debug_once( - "%s does not hold its eps (%s) in an attribute. Every instance in this " - "model will use the value traced from this instance. If this is not " - "desired, consider storing and reading eps using attribute of %s.", - type(module).__name__, - eps, - type(module).__name__, - ) - return None, eps - - @staticmethod - def _eps_from_graph(graph: fx.Graph) -> float | None: - """Extract the `eps` constant from the graph, if present.""" - if (x := find_node(graph, lambda n: n.op == "placeholder")) is None: - return None - fused = find_node(graph, lambda n: is_op(n, "rms_norm")) - if fused is not None and peel(_operand(fused, 0, "input")) is x: - args, kwargs = fused.args, fused.kwargs - eps = args[3] if len(args) > 3 else kwargs.get("eps") - return float(eps) if isinstance(eps, (int, float)) else None - for node in graph.nodes: - if _is_inverse_sqrt(node) and (eps := _variance_eps(node, x)) is not None: - return eps - return None - - def validate(self, module: nn.Module, vllm_config: "VllmConfig") -> bool: - return True - - def fuse( - self, module: nn.Module, prefix: str, vllm_config: "VllmConfig" - ) -> nn.Module: - """Fuse the matched RMSNorm pattern into a vLLM fused RMSNorm CustomOp.""" - weight = getattr(module, "weight", None) - hidden_size = weight.size(0) if weight is not None else 0 - eps = getattr(module, self.eps_attr, None) if self.eps_attr else self.eps - if not isinstance(eps, (int, float)): - # If eps was not detected, match torch behaviour. - config_dtype = vllm_config.model_config.dtype - dtype = weight.dtype if weight is not None else config_dtype - eps = torch.finfo(dtype).eps - if self.zero_centered: - return TPAwareGemmaRMSNorm(hidden_size=hidden_size, eps=eps) - return TPAwareRMSNorm( - hidden_size=hidden_size, - eps=eps, - has_weight=weight is not None, - dtype=weight.dtype if weight is not None else None, - ) diff --git a/vllm/model_executor/models/transformers/fx_utils.py b/vllm/model_executor/models/transformers/fx_utils.py deleted file mode 100644 index 818e1ca7bb..0000000000 --- a/vllm/model_executor/models/transformers/fx_utils.py +++ /dev/null @@ -1,818 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""fx tracing and forward-source rewriting for the Transformers backend fusers. - -A small engine, independent of any particular pattern: trace a module's forward -with `torch.fx` (tolerating a partial graph), inspect the resulting nodes, and -rewrite the forward's *source* (AST) so only matched calls change while the rest -stays live Python. `fusion.py` builds the concrete fusion patterns on top. -""" - -import ast -import contextlib -import inspect -import operator -import textwrap -from collections.abc import Callable -from itertools import chain -from typing import Any -from unittest import mock - -import torch -from torch import fx, nn -from torch.nn import functional as F -from typing_extensions import TypeIs - -from vllm.logger import init_logger - -logger = init_logger(__name__) - -_UNKNOWN: Any = object() -"""Sentinel meta value for proxies whose concrete value could not be inferred. -Distinct from `None`, which is a valid concrete value (e.g. `attn_weights`).""" - -_MODULE_CALL = nn.Module.__call__ -"""The unpatched `nn.Module.__call__`. During tracing fx patches it to record -`call_module` nodes; meta execution must call modules for real.""" - -_MODULE_GETATTR = nn.Module.__getattr__ -"""The unpatched `nn.Module.__getattr__`. During tracing fx patches it to record -`get_attr` nodes; reading one while inferring a `get_attr`'s own value would -recurse until the stack runs out, truncating the graph.""" - - -@contextlib.contextmanager -def _unpatched_modules(): - """Restores the module hooks fx patches, so meta execution records nothing.""" - with ( - mock.patch.object(nn.Module, "__call__", _MODULE_CALL), - mock.patch.object(nn.Module, "__getattr__", _MODULE_GETATTR), - ): - yield - - -def is_leaf_call(node: object) -> TypeIs[fx.Node]: - """Is node a call recorded by `_as_leaf_call` (e.g. an attention interface).""" - return isinstance(node, fx.Node) and node.meta.get("leaf_call", False) - - -def _reference_weight(module: nn.Module) -> torch.Tensor | None: - """A weight whose trailing dim is the module's hidden size. - - Linears and 2-D gate weights are `[out, hidden]`; norm weights are - `[hidden]`. Used to fabricate a placeholder input of matching size/dtype.""" - for child in module.modules(): - if isinstance(child, nn.Linear): - return child.weight - for param in module.parameters(): - if param.ndim in (1, 2): - return param - return None - - -class _MetaProxy(fx.Proxy): - """Proxy carrying the meta-tensor value of the traced expression. - - Shape questions (`len`, iteration, `.shape` unpacks) are answered by - executing each op on the meta values, so PyTorch's meta kernels are the - single source of shape inference — no per-op rules.""" - - meta: Any = _UNKNOWN - - def __len__(self) -> int: - if self.meta is not _UNKNOWN: - return len(self.meta) - return super().__len__() # type: ignore[misc] - - def __getattr__(self, k: str) -> "_MetaAttribute": - return _MetaAttribute(self, k) - - def __setitem__(self, key: object, value: object) -> None: - # Record `obj[key] = value` so tracing continues past in-place writes - # into containers (e.g. Gemma 4's `shared_kv_states[...] = ...`). - self.tracer.create_proxy( - "call_function", operator.setitem, (self, key, value), {} - ) - - -class _MetaAttribute(_MetaProxy, fx.proxy.Attribute): - """Attribute proxy (e.g. `x.shape`) carrying its meta value. - - `Proxy.__getattr__` constructs `Attribute` directly, bypassing - `Tracer.proxy`, so the meta value must be grafted on here too.""" - - def __init__(self, root: fx.Proxy, attr: str): - super().__init__(root, attr) - root_meta = getattr(root, "meta", _UNKNOWN) - if root_meta is not _UNKNOWN: - with contextlib.suppress(Exception): - self.meta = getattr(root_meta, attr) - - -class _AllLeafTracer(fx.Tracer): - """Tracer that treats every submodule as a leaf. - - Each child stays one `call_module` node, so matching sees the module's own - forward structure (activations aren't decomposed into e.g. `sigmoid * x`). - Every traced op is also executed on meta tensors (see `_MetaProxy`) so - shape unpacks and `*`-splats trace through; anything else untraceable ends - the trace early and the partial graph is matched. - """ - - varkw: str | None = None - """Name of the traced forward's `**kwargs` parameter, if any.""" - - def is_leaf_module(self, m: nn.Module, module_qualified_name: str) -> bool: - return True - - def proxy(self, node: fx.Node) -> fx.Proxy: - return _MetaProxy(node, self) - - def create_proxy(self, kind, target, args, kwargs, *extra, **extra_kwargs): - proxy = super().create_proxy(kind, target, args, kwargs, *extra, **extra_kwargs) - if isinstance(proxy, _MetaProxy) and proxy.meta is _UNKNOWN: - # A failure stays _UNKNOWN; only fatal if a shape question is asked. - with contextlib.suppress(Exception): - proxy.meta = self._infer_meta(kind, target, args, kwargs) - return proxy - - def _infer_meta(self, kind: str, target: object, args: tuple, kwargs: dict): - """Execute the op on meta tensors; PyTorch infers the output value.""" - if kind == "placeholder": - # vLLM always feeds the model [1, seq_len, hidden_size] hidden states. - weight = _reference_weight(self.root) - if str(target) == "hidden_states" and weight is not None: - return torch.empty( - 1, 8, weight.shape[-1], dtype=weight.dtype, device="meta" - ) - return _UNKNOWN - if kind == "get_attr": - with _unpatched_modules(): - value = operator.attrgetter(str(target))(self.root) - if isinstance(value, torch.Tensor): - value = torch.empty_like(value, device="meta") - return value - unknown = False - - def meta_of(arg: object) -> object: - nonlocal unknown - if isinstance(arg, fx.Proxy): - meta = getattr(arg, "meta", _UNKNOWN) - unknown = unknown or meta is _UNKNOWN - return meta - return arg - - meta_args = fx.node.map_aggregate(args, meta_of) - meta_kwargs = fx.node.map_aggregate(kwargs, meta_of) - if unknown: - return _UNKNOWN - if kind == "call_function": - assert callable(target) - return target(*meta_args, **meta_kwargs) - if kind == "call_method": - receiver, *rest = meta_args - return getattr(receiver, str(target))(*rest, **meta_kwargs) - if kind == "call_module": - # Run the child's forward with all its state on "meta", without - # mutating it (at match time params may be meta but buffers real). - child = self.root.get_submodule(str(target)) - state = { - name: torch.empty_like(tensor, device="meta") - for name, tensor in chain( - child.named_parameters(), child.named_buffers() - ) - } - with _unpatched_modules(): - return torch.func.functional_call(child, state, meta_args, meta_kwargs) - return _UNKNOWN - - def _is_varkw(self, node: object) -> bool: - return ( - isinstance(node, fx.Node) - and node.op == "placeholder" - and str(node.target).lstrip("*") == self.varkw - ) - - def iter(self, obj: fx.Proxy): - # Assume kwargs is always empty to simplify tracing. - node = obj.node - if self._is_varkw(node) or ( - node.op == "call_method" - and node.target == "keys" - and self._is_varkw(node.args[0]) - ): - return iter(()) - meta = getattr(obj, "meta", _UNKNOWN) - if meta is _UNKNOWN: - return super().iter(obj) - return iter([obj[i] for i in range(len(meta))]) - - -def _as_leaf_call(fn: Callable, length: int | None = None) -> Callable: - """Wrap any callable so tracing records it as one opaque `call_function` node. - - Lets the trace continue past untraceable bodies. Only the proxy arguments carry into - the node's dataflow; the rest are dropped rather than lifted into the graph. - `length` declares how many values the callable returns, so unpacking its result also - traces. Called without proxies (i.e. outside tracing), the wrapper is a passthrough. - """ - - def leaf(*args, **kwargs): - proxies = tuple(arg for arg in args if isinstance(arg, fx.Proxy)) - if not proxies: - return fn(*args, **kwargs) - proxy = proxies[0].tracer.create_proxy("call_function", fn, proxies, {}) - proxy.node.meta["leaf_call"] = True - if length is not None: - # The body never executes, so fabricate a value of the declared length. - proxy.meta = (_UNKNOWN,) * length - return proxy - - return leaf - - -def _leaf_attention_interfaces(): - """Patch `AttentionInterface.get_interface` so traced forwards see a leaf node. - - `vllm_attention_function` needs runtime context so it is untraceable. - Every interface returns `(attn_output, attn_weights)`.""" - from transformers.modeling_utils import AttentionInterface - - original = AttentionInterface.get_interface - - def get_interface(self, *args, **kwargs): - return _as_leaf_call(original(self, *args, **kwargs), length=2) - - return mock.patch.object(AttentionInterface, "get_interface", get_interface) - - -def trace(module: nn.Module) -> fx.Graph | None: - """Trace `module.forward`, returning the partial graph on failure.""" - parameters = forward_parameters(type(module)) - # vLLM never passes `past_key_values` so it is always the default value of `None`. - # Make this concrete to simplify tracing. - concrete_args = None - if "past_key_values" in parameters: - concrete_args = {"past_key_values": None} - # Get the name of the kwargs parameter passed to module.forward (usually "kwargs") - tracer = _AllLeafTracer() - tracer.varkw = next( - ( - p.name - for p in parameters.values() - if p.kind is inspect.Parameter.VAR_KEYWORD - ), - None, - ) - try: - with _leaf_attention_interfaces(): - return tracer.trace(module, concrete_args=concrete_args) - except Exception as exc: - logger.debug("Could not fully trace %s: %s", type(module), exc) - return getattr(tracer, "graph", None) - - -def recover_forward(cls: type[nn.Module]) -> tuple[ast.FunctionDef, Callable]: - """Parse the source of `cls.forward`, ready for rewriting.""" - fn = inspect.unwrap(cls.forward) - if fn.__code__.co_freevars: - raise ValueError("forward is a closure") - tree = ast.parse(textwrap.dedent(inspect.getsource(fn))) - funcdef = tree.body[0] - if not isinstance(funcdef, ast.FunctionDef): - raise ValueError("source is not a plain function definition") - # `fn` is already unwrapped; don't re-apply its decorators - funcdef.decorator_list.clear() - # Annotations may not evaluate outside the defining module (e.g. with - # postponed evaluation); they're not needed at runtime - funcdef.returns = None - args = funcdef.args - for arg in ( - *args.posonlyargs, - *args.args, - *args.kwonlyargs, - *filter(None, (args.vararg, args.kwarg)), - ): - arg.annotation = None - # Recompiling outside the class body would break name mangling - for node in ast.walk(funcdef): - name = getattr(node, "attr", None) or getattr(node, "id", None) - if name and name.startswith("__") and not name.endswith("__"): - raise ValueError(f"{name} would be name mangled") - return funcdef, fn - - -def forward_parameters(cls: type[nn.Module]) -> dict[str, inspect.Parameter]: - """`cls.forward`'s signature parameters, or empty if uninspectable.""" - try: - return dict(inspect.signature(cls.forward).parameters) - except (TypeError, ValueError): - return {} - - -def forward_input_count(cls: type[nn.Module]) -> int: - """The number of tensor inputs `cls.forward` declares, excluding `self` and - any `*args`/`**kwargs`. Read from the signature, so it is independent of - whether the trace completes (unlike counting placeholders).""" - params = list(forward_parameters(cls).values()) - if not params: - return 1 # uninspectable: assume a single input and let matching decide - fixed = ( - inspect.Parameter.POSITIONAL_ONLY, - inspect.Parameter.POSITIONAL_OR_KEYWORD, - inspect.Parameter.KEYWORD_ONLY, - ) - return sum(1 for p in params[1:] if p.kind in fixed) - - -def compile_forward(funcdef: ast.FunctionDef, fn: Callable) -> Callable: - """Compile `funcdef` in `fn`'s module so tracebacks point at the source.""" - module = ast.Module(body=[funcdef], type_ignores=[]) - ast.fix_missing_locations(module) - ast.increment_lineno(module, fn.__code__.co_firstlineno - 1) - code = compile(module, fn.__code__.co_filename, "exec") - namespace: dict = {} - exec(code, fn.__globals__, namespace) - return namespace[funcdef.name] - - -def self_call_and_refs( - funcdef: ast.FunctionDef, name: str -) -> tuple[ast.Call, list[ast.Attribute]]: - """The unique `self.(arg)` call, plus every other `self.` reference. - - Raises unless exactly one single-argument `self.(arg)` call exists, so - one fx `call_module` node maps to one syntactic call site. The remaining - references (e.g. `is not None` guards) are returned for the caller to resolve. - """ - uses = [ - node - for node in ast.walk(funcdef) - if isinstance(node, ast.Attribute) - and node.attr == name - and isinstance(node.value, ast.Name) - and node.value.id == "self" - ] - calls = [ - node - for node in ast.walk(funcdef) - if isinstance(node, ast.Call) - and any(node.func is use for use in uses) - and len(node.args) == 1 - and not isinstance(node.args[0], ast.Starred) - and not node.keywords - ] - if len(calls) != 1: - raise ValueError(f"{name} is not called exactly once as self.{name}(arg)") - call = calls[0] - other_refs = [use for use in uses if use is not call.func] - return call, other_refs - - -def single_self_call(funcdef: ast.FunctionDef, name: str) -> ast.Call: - """The unique `self.(arg)` call in `funcdef`. - - Raises unless `name` appears exactly once, as such a call, so the source - rewrite agrees with the fx match. - """ - call, other_refs = self_call_and_refs(funcdef, name) - if other_refs: - raise ValueError(f"{name} is referenced {len(other_refs) + 1} times") - return call - - -def _is_none(node: ast.expr) -> bool: - return isinstance(node, ast.Constant) and node.value is None - - -def _in_boolean_context(funcdef: ast.FunctionDef, ref: ast.expr) -> bool: - """Is `ref` used only for its truth value (a test, or a `not` operand)? - - In these positions the object's identity never escapes, so a reference that - is always truthy can be replaced by `True`. `and`/`or` are excluded: they - yield an operand, so the module could escape (`x and self.`).""" - for node in ast.walk(funcdef): - if ( - isinstance(node, (ast.If, ast.IfExp, ast.While, ast.Assert)) - and node.test is ref - ): - return True - if ( - isinstance(node, ast.UnaryOp) - and isinstance(node.op, ast.Not) - and node.operand is ref - ): - return True - return False - - -def bypass_existence_guard( - funcdef: ast.FunctionDef, ref: ast.Attribute, name: str -) -> None: - """Fold a guard on `self.`'s existence to its constant value. - - The fuser deletes `self.` and binds only instances where it exists as a - truthy `nn.Linear` (guaranteed by the match, the cache key, and `validate`), - so a guard testing its presence is a fusion invariant. Two forms are folded: - - - an identity `None` check `self. is None` -> `False`, - `self. is not None` -> `True` (either operand order). `==`/`!=` are - *not* folded: `is_linear` accepts `nn.Linear` subclasses, which may override - `__eq__`/`__ne__`, so equality is not guaranteed to track identity. - - a bare truthiness test (`if self.:`, `... if self. else ...`, - `not self.`): the reference itself to `True`. - - Any other surviving reference escapes the projection's value, which no longer - exists after fusion, so refuse rather than change semantics.""" - for node in ast.walk(funcdef): - # `self. is (not) None`, either operand order. - if not (isinstance(node, ast.Compare) and len(node.ops) == 1): - continue - (op,), (right,) = node.ops, node.comparators - if not isinstance(op, (ast.Is, ast.IsNot)): - continue - if (ref is node.left and _is_none(right)) or ( - ref is right and _is_none(node.left) - ): - value = isinstance(op, ast.IsNot) - replace_expr(funcdef, node, ast.copy_location(ast.Constant(value), node)) - return - # A bare truthiness test (statement `if`, ternary, or `not`); an `nn.Linear` - # is always truthy, so the reference folds to `True`. - if _in_boolean_context(funcdef, ref): - replace_expr(funcdef, ref, ast.copy_location(ast.Constant(True), ref)) - return - raise ValueError(f"{name} is referenced outside an existence guard") - - -def block_chain( - block: list[ast.stmt], node: ast.AST -) -> list[tuple[list[ast.stmt], int]]: - """Path of (statement list, index) pairs from `block` down to `node`. - - Each pair names a nested block and the index in it of the statement - containing `node`; the last pair is `node`'s innermost block. Empty if - `node` is not in `block`. The longest common prefix of several nodes' chains - is the innermost block that dominates them all. - """ - for index, stmt in enumerate(block): - if not any(child is node for child in ast.walk(stmt)): - continue - child_blocks = [ - getattr(stmt, fld, None) for fld in ("body", "orelse", "finalbody") - ] - child_blocks += [h.body for h in getattr(stmt, "handlers", [])] - child_blocks += [c.body for c in getattr(stmt, "cases", [])] - for child_block in child_blocks: - if ( - isinstance(child_block, list) - and child_block - and (tail := block_chain(child_block, node)) - ): - return [(block, index), *tail] - return [(block, index)] - return [] - - -def _base_name(node: ast.expr) -> str | None: - """The root `Name` of an attribute/subscript chain (`a.b[c]` -> `a`).""" - while isinstance(node, (ast.Attribute, ast.Subscript)): - node = node.value - return node.id if isinstance(node, ast.Name) else None - - -def _rebound_names(region: list[ast.stmt]) -> set[str]: - """Names rebound outright within `region` (`x = ...`, `del x`). - - These are `Name` nodes in a `Store`/`Del` context.""" - return { - node.id - for stmt in region - for node in ast.walk(stmt) - if isinstance(node, ast.Name) and isinstance(node.ctx, (ast.Store, ast.Del)) - } - - -def _inplace_target(node: ast.AST) -> str | None: - """Base name `node` mutates in place, if any. - - A write through the name (`x[i] = ...`, `x.attr = ...`) or an in-place method - call (`x.mul_(...)`) leaves the base name in a `Load` context, so it is not a - plain `Name` store (see `_rebound_names`).""" - if isinstance(node, (ast.Attribute, ast.Subscript)) and isinstance( - node.ctx, (ast.Store, ast.Del) - ): - return _base_name(node) - if ( - isinstance(node, ast.Call) - and isinstance(node.func, ast.Attribute) - and node.func.attr.endswith("_") - and not node.func.attr.endswith("__") - ): - return _base_name(node.func.value) - return None - - -def _mutated_names(region: list[ast.stmt]) -> set[str]: - """Names mutated in place within `region` (see `_inplace_target`).""" - return { - base - for stmt in region - for node in ast.walk(stmt) - if (base := _inplace_target(node)) is not None - } - - -def _call_name(call: ast.Call) -> str: - """The called function's own name, without its qualifier.""" - if isinstance(call.func, ast.Attribute): - return call.func.attr - return call.func.id if isinstance(call.func, ast.Name) else "" - - -def _arg_names(call: ast.Call) -> set[str]: - """Every name `call` reads through its arguments.""" - reads = [*call.args, *(kw.value for kw in call.keywords)] - return { - node.id - for arg in reads - for node in ast.walk(arg) - if isinstance(node, ast.Name) and isinstance(node.ctx, ast.Load) - } - - -def _writes_through_args(call: ast.Call) -> bool: - """Does `call` write through an argument rather than return a new value? - - Torch spells this three ways: an `out=` destination, `inplace=True`, and the - trailing-underscore convention (`torch.relu_(x)`). The underscore is checked - on the function's own name, so it catches the free-function form that - `_inplace_target` cannot (there the base name is the module, `torch`). - """ - for keyword in call.keywords: - if keyword.arg == "out": - return True - if keyword.arg == "inplace" and not ( - isinstance(keyword.value, ast.Constant) and keyword.value.value is False - ): - return True - name = _call_name(call) - return name.endswith("_") and not name.endswith("__") - - -def written_names(region: list[ast.stmt]) -> set[str]: - """Names `region` may rebind or write through. - - Beyond plain rebinding and writes through the name itself (`_rebound_names`, - `_mutated_names`), a call can write through an argument without naming it as - a target: `F.relu(x, inplace=True)`, `torch.add(x, y, out=x)`, - `torch.relu_(x)`. A statement-level call whose result is discarded is treated - the same way, since it can only be there for a side effect. - """ - written = _rebound_names(region) | _mutated_names(region) - for stmt in region: - if isinstance(stmt, ast.Expr) and isinstance(stmt.value, ast.Call): - written |= _arg_names(stmt.value) - for node in ast.walk(stmt): - if isinstance(node, ast.Call) and _writes_through_args(node): - written |= _arg_names(node) - return written - - -_METADATA_ATTRS = frozenset( - {"shape", "dtype", "device", "ndim", "size", "dim", "numel", "item"} -) - - -def _aliasing_reads(node: ast.expr) -> set[str]: - """Names `node` reads in a way that could yield a view of them. - - Metadata (`x.shape`, `x.size(0)`, `x.dtype`) is ints and tuples sharing no - storage with `x`, so reaching a name only through metadata cannot alias it. - Shape arithmetic is pervasive between projections, so propagating through it - would taint nearly every later name. - """ - names: set[str] = set() - stack: list[ast.AST] = [node] - while stack: - current = stack.pop() - if isinstance(current, ast.Attribute) and current.attr in _METADATA_ATTRS: - continue - if ( - isinstance(current, ast.Call) - and isinstance(current.func, ast.Attribute) - and current.func.attr in _METADATA_ATTRS - ): - stack.extend(current.args) - stack.extend(keyword.value for keyword in current.keywords) - continue - if isinstance(current, ast.Name) and isinstance(current.ctx, ast.Load): - names.add(current.id) - stack.extend(ast.iter_child_nodes(current)) - return names - - -def _returns_fresh(node: ast.expr, module: nn.Module) -> bool: - """Does evaluating `node` allocate, rather than return a view of its input? - - A `nn.Linear` writes its output into new storage, so its result aliases - nothing that went in, and a chain of view ops on top of it aliases only that - fresh tensor. Walking the receiver chain down to a `self.(...)` call - therefore proves the value cannot alias the projection input. A literal - aliases nothing either, and a branch or sequence is fresh when every part is - -- which is how a guarded projection (`p(x) if p is not None else None`) - stays untracked. A call on an attribute that is missing or `None` cannot run - at all, so a branch selecting a different configuration is fresh too. - """ - if isinstance(node, ast.Constant): - return True - if isinstance(node, ast.IfExp): - if isinstance(node.test, ast.Constant): - # A folded existence guard leaves the untaken branch as dead code. - taken = node.body if node.test.value else node.orelse - return _returns_fresh(taken, module) - return _returns_fresh(node.body, module) and _returns_fresh(node.orelse, module) - if isinstance(node, (ast.Tuple, ast.List)): - return all(_returns_fresh(elt, module) for elt in node.elts) - while isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute): - func = node.func - if isinstance(func.value, ast.Name) and func.value.id == "self": - attribute = getattr(module, func.attr, None) - if attribute is None or isinstance(attribute, nn.Linear): - # A missing or `None` attribute cannot be called, so the branch - # selecting it never runs (`q_proj` on a q-LoRA layer). - return True - # Anything else can only return storage it was handed, so it is - # fresh exactly when its inputs are (`q_norm(q_proj(x))`). - return bool(node.args) and all( - _returns_fresh(arg, module) for arg in node.args - ) - node = func.value - return False - - -def aliasing_names( - funcdef: ast.FunctionDef, seed: set[str], module: nn.Module -) -> set[str]: - """`seed` plus every name that may alias one of them. - - A view shares storage with its base (`h = x.view(-1)`), so mutating `h` - mutates `x`; tracking only `seed` would miss that. Any assignment reading a - tracked name therefore taints its targets, transitively, unless its value is - freshly allocated (`_returns_fresh`). - """ - names = set(seed) - assigns = [ - (node, node.value) - for node in ast.walk(funcdef) - if isinstance(node, (ast.Assign, ast.AnnAssign)) - and node.value is not None - and not _returns_fresh(node.value, module) - ] - changed = True - while changed: - changed = False - for node, value in assigns: - if not _aliasing_reads(value) & names: - continue - targets = node.targets if isinstance(node, ast.Assign) else [node.target] - stores = { - child.id - for target in targets - for child in ast.walk(target) - if isinstance(child, ast.Name) and isinstance(child.ctx, ast.Store) - } - if stores - names: - names |= stores - changed = True - return names - - -def replace_expr(module: ast.AST, old: ast.expr, new: ast.expr) -> None: - """Replace the expression `old` (by identity) with `new` within `module`.""" - - class _Replacer(ast.NodeTransformer): - def visit(self, node: ast.AST) -> ast.AST: - if node is old: - return new - return super().generic_visit(node) - - _Replacer().visit(module) - - -def is_complete(graph: fx.Graph) -> bool: - """Did the trace run to completion? A partial graph has no `output` node. - - Only a complete graph can be read for the *absence* of an op; in a partial - one the tail is exactly what is missing.""" - return any(node.op == "output" for node in graph.nodes) - - -def find_node(graph: fx.Graph, predicate: Callable[[fx.Node], bool]) -> fx.Node | None: - """The first node in `graph` matching `predicate`, or `None`.""" - return next((n for n in graph.nodes if predicate(n)), None) - - -def output_value(graph: fx.Graph) -> object | None: - """The value the graph's `output` node returns, if the trace reached one.""" - output = find_node(graph, lambda n: n.op == "output") - if output is None or not output.args: - return None - return output.args[0] - - -def upstream_linear(node: object, module: nn.Module) -> fx.Node | None: - """Nearest linear producing `node`, walking back through splits/reshapes. - - Non-linear submodules are transparent too (e.g. the dropout GPT-style - attentions apply after their output projection). Never walks through a leaf - call (e.g. an attention interface): its inputs are what attention consumes, - not what produced the value.""" - stack = [node] - seen: set[fx.Node] = set() - while stack: - current = stack.pop() - if not isinstance(current, fx.Node) or current in seen: - continue - seen.add(current) - if is_linear(current, module): - return current - if current.op in ( - "call_function", - "call_method", - "call_module", - ) and not is_leaf_call(current): - stack.extend(current.args) - return None - - -def downstream_linear(node: fx.Node, module: nn.Module) -> fx.Node | None: - """Nearest linear consuming `node`'s output, walking through casts/scalings. - - Never walks through a leaf call (e.g. an attention interface): what crosses - it is consumed by the attention computation, not projected.""" - queue = list(node.users) - seen: set[fx.Node] = set() - while queue: - current = queue.pop(0) - if current in seen: - continue - seen.add(current) - if is_linear(current, module): - return current - if current.op in ("call_function", "call_method") and not is_leaf_call(current): - queue.extend(current.users) - return None - - -def returned_linear(graph: fx.Graph, module: nn.Module) -> str | None: - """Name of the Linear producing the graph's (first) output value.""" - value = output_value(graph) - if isinstance(value, (tuple, list)) and value: - value = value[0] - linear = upstream_linear(value, module) - return None if linear is None else str(linear.target) - - -def is_linear(node: fx.Node, module: nn.Module) -> bool: - """Is node `nn.Linear.__call__()`.""" - return node.op == "call_module" and isinstance( - module.get_submodule(node.target), nn.Linear - ) - - -_DTYPE_CASTS = frozenset({"to", "float", "double", "half", "bfloat16", "type_as"}) - - -def peel(node: object) -> object: - """Strip dtype-cast wrappers (`.to(...)`, `.float()`, `.type_as(...)`).""" - while ( - isinstance(node, fx.Node) - and node.op == "call_method" - and node.target in _DTYPE_CASTS - ): - node = node.args[0] - return node - - -def is_fn(node: object, target: object) -> TypeIs[fx.Node]: - """Is node `()`.""" - return ( - isinstance(node, fx.Node) - and node.op == "call_function" - and node.target is target - ) - - -def is_method(node: object, name: str) -> TypeIs[fx.Node]: - """Is node `.()`.""" - return ( - isinstance(node, fx.Node) and node.op == "call_method" and node.target == name - ) - - -def is_op(node: object, name: str) -> TypeIs[fx.Node]: - """Is node `.()` for torch, F, operator, or Tensor.""" - return any( - is_fn(node, getattr(module, name, None)) for module in (torch, F, operator) - ) or (hasattr(torch.Tensor, name) and is_method(node, name)) diff --git a/vllm/model_executor/models/transformers/layers.py b/vllm/model_executor/models/transformers/layers.py deleted file mode 100644 index 4ff3651dfc..0000000000 --- a/vllm/model_executor/models/transformers/layers.py +++ /dev/null @@ -1,64 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Layer provider resolution for the Transformers modeling backend. - -When ``VLLM_USE_HW_AGNOSTIC`` is set, layer symbols are imported from -``vllm.model_executor.hw_agnostic.layers.``, falling back to -``vllm.model_executor.layers.`` for anything not yet ported. The -resolved source of every symbol is logged so it is clear which layers run -hw-agnostic and which fell back to vLLM. -""" - -import importlib - -import vllm.envs as envs -from vllm.logger import init_logger - -logger = init_logger(__name__) - -_HW_PKG = "vllm.model_executor.hw_agnostic.layers" -_VLLM_PKG = "vllm.model_executor.layers" - - -def _resolve(module: str, name: str): - """Return `name` from the hw-agnostic `module` when enabled and available, - else from vLLM. Logs which source was used.""" - if envs.VLLM_USE_HW_AGNOSTIC: - try: - obj = getattr(importlib.import_module(f"{_HW_PKG}.{module}"), name) - logger.info("Using hw-agnostic layer: %s", name) - return obj - except (ImportError, AttributeError): - logger.warning( - "hw-agnostic layer %s is not available; falling back to default", name - ) - return getattr(importlib.import_module(f"{_VLLM_PKG}.{module}"), name) - - -RMSNorm = _resolve("layernorm", "RMSNorm") -GemmaRMSNorm = _resolve("layernorm", "GemmaRMSNorm") - - -def get_act_and_mul_fn(act_fn_name: str): - """Fused activation-and-mul op for `act_fn_name`, preferring hw-agnostic. - - Resolved per call because the op is name-parameterized: an activation with - no hw-agnostic equivalent falls back to vLLM individually. - """ - if envs.VLLM_USE_HW_AGNOSTIC: - try: - from vllm.model_executor.hw_agnostic.layers.activation import ( - get_act_and_mul_fn as hw_fn, - ) - - fn = hw_fn(act_fn_name) - logger.info_once("Using hw-agnostic activation: %s", act_fn_name) - return fn - except (ImportError, KeyError): - logger.warning_once( - "hw-agnostic activation %s is not available; falling back to vLLM", - act_fn_name, - ) - from vllm.model_executor.layers.activation import get_act_and_mul_fn as vllm_fn - - return vllm_fn(act_fn_name) diff --git a/vllm/model_executor/models/transformers/legacy.py b/vllm/model_executor/models/transformers/legacy.py deleted file mode 100644 index 7a256f29d5..0000000000 --- a/vllm/model_executor/models/transformers/legacy.py +++ /dev/null @@ -1,80 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -# Copyright 2024 The vLLM team. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Transformers modeling backend mixin for legacy models.""" - -from typing import TYPE_CHECKING - -import torch - -from vllm.model_executor.models.utils import WeightsMapper -from vllm.sequence import IntermediateTensors - -from .base import Base - -if TYPE_CHECKING: - from vllm.config import VllmConfig - - -class LegacyMixin(Base): - def __init__(self, *, vllm_config: "VllmConfig", prefix: str = ""): - super().__init__(vllm_config=vllm_config, prefix=prefix) - - self.hf_to_vllm_mapper |= WeightsMapper( - # Drop unsupported/unwanted output embeddings layers. - orig_to_new_prefix={ - "model.lm_head.": None, - "model.predictions.": None, - "model.qa_outputs.": None, - "model.embeddings_project.": None, - "model.discriminator_predictions.": None, - }, - orig_to_new_substr={ - # Some encoder models have the position_ids buffer in the checkpoint. - # vLLM always passes position_ids as an argument, so drop the buffer. - "position_ids": None, - # Some encoder models have the bias of the final classifier layer in - # the checkpoint. vLLM does not use this bias, so drop it. - "score.bias": None, - }, - ) - - # roberta-like models an extra padding in positions. - # FIXME(Isotr0py): This is quite hacky for roberta edge case, - # we should find a better way to handle this. - self.is_roberta = "roberta" in self.text_config.model_type - self.padding_idx = self.text_config.pad_token_id - - def forward( - self, - input_ids: torch.Tensor | None, - positions: torch.Tensor, - intermediate_tensors: IntermediateTensors | None = None, - inputs_embeds: torch.Tensor | None = None, - **kwargs, - ) -> torch.Tensor | IntermediateTensors: - if self.is_roberta: - # RoBERTa positions start at padding_idx + 1. - # Non-in-place add to avoid mutating the persistent GPU buffer -- - # in-place += would accumulate on CUDA graph padding slots. - positions = positions + self.padding_idx + 1 - return super().forward( - input_ids=input_ids, - positions=positions, - intermediate_tensors=intermediate_tensors, - inputs_embeds=inputs_embeds, - **kwargs, - ) diff --git a/vllm/model_executor/models/transformers/moe.py b/vllm/model_executor/models/transformers/moe.py deleted file mode 100644 index a426a2c9d9..0000000000 --- a/vllm/model_executor/models/transformers/moe.py +++ /dev/null @@ -1,454 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -# Copyright 2024 The vLLM team. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Transformers modeling backend mixin for Mixture of Experts (MoE) models.""" - -from dataclasses import dataclass -from functools import partial -from typing import TYPE_CHECKING, Any - -import torch -import torch.nn as nn - -from vllm._aiter_ops import rocm_aiter_ops -from vllm.config.utils import getattr_iter -from vllm.distributed import get_dp_group, get_ep_group -from vllm.forward_context import ForwardContext, get_forward_context -from vllm.logger import init_logger -from vllm.model_executor.custom_op import PluggableLayer -from vllm.model_executor.layers.fused_moe import ( - FusedMoEFactory, - MoERunner, - RoutedExperts, -) -from vllm.model_executor.models.interfaces import MixtureOfExperts -from vllm.model_executor.models.transformers.fuser import get_fuser -from vllm.model_executor.models.transformers.fusers.glu import GLUFuser -from vllm.model_executor.models.transformers.fusers.moe import MoEBlockFuser -from vllm.model_executor.models.utils import extract_layer_index, maybe_prefix -from vllm.transformers_utils.config import CONFIG_FIELD_ALIASES -from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE, direct_register_custom_op - -from .base import Base -from .utils import log_replacement, maybe_per_layer - -if TYPE_CHECKING: - from vllm.config import VllmConfig - -logger = init_logger(__name__) - - -@dataclass -class TransformersMoEState: - topk_ids: torch.Tensor | None = None - is_sequence_parallel: bool = False - - -# --8<-- [start:transformers_fused_moe] -@PluggableLayer.register("transformers_fused_moe") -class TransformersMoERunner(MoERunner): - """Custom MoERunner for the Transformers modeling backend.""" - - # --8<-- [end:transformers_fused_moe] - def __init__(self, *args, moe_state: TransformersMoEState, **kwargs): - super().__init__(*args, **kwargs) - self._moe_state = moe_state - self._moe_state.is_sequence_parallel = self.moe_config.is_sequence_parallel - - def forward( # type: ignore[override] - self, - hidden_states: torch.Tensor, - topk_ids: torch.Tensor, - topk_weights: torch.Tensor, - **kwargs: Any, - ) -> torch.Tensor: - """In Transformers `experts.forward` will have this signature, which is why - it deliberately does not match `MoERunner.forward`. - - We discard any extra kwargs because we cannot use them here.""" - # Note: we need to forward through a custom op so the topk_ids - # can be transferred without interfering with cudagraphs. - return torch.ops.vllm.transformers_moe_forward( - hidden_states, - topk_ids.to(torch.int32), - topk_weights.to(torch.float32), - self.layer_name, - ) - - def _forward_super( - self, - hidden_states: torch.Tensor, - topk_weights: torch.Tensor, - ) -> torch.Tensor: - return super().forward(hidden_states, topk_weights) - - -def _transformers_moe_forward( - hidden_states: torch.Tensor, - topk_ids: torch.Tensor, - topk_weights: torch.Tensor, - layer_name: str, -) -> torch.Tensor: - """Store the `topk_ids` in the layer and call the actual forward.""" - forward_context: ForwardContext = get_forward_context() - self = forward_context.no_compile_layers[layer_name] - self._moe_state.topk_ids = topk_ids - return self._forward_super(hidden_states, topk_weights) - - -def _transformers_moe_forward_fake( - hidden_states: torch.Tensor, - topk_ids: torch.Tensor, - topk_weights: torch.Tensor, - layer_name: str, -) -> torch.Tensor: - return torch.empty_like(hidden_states) - - -direct_register_custom_op( - op_name="transformers_moe_forward", - op_func=_transformers_moe_forward, - mutates_args=["hidden_states"], - fake_impl=_transformers_moe_forward_fake, - tags=(torch.Tag.needs_fixed_stride_order,), -) - - -def _hf_topk_router_precomputed( - hidden_states: torch.Tensor, - gating_output: torch.Tensor, - topk: int, - renormalize: bool, - moe_state: TransformersMoEState, -) -> tuple[torch.Tensor, torch.Tensor]: - """Return `(topk_weights, topk_ids)` the HF block forward already computed: - weights arrive as `gating_output`, ids via `moe_state`.""" - topk_weights = gating_output - topk_ids = moe_state.topk_ids - assert topk_ids is not None - # Handle all gather in expert parallel - if topk_ids.size(0) != hidden_states.size(0): - dp_metadata = get_forward_context().dp_metadata - assert dp_metadata is not None - sizes = dp_metadata.get_chunk_sizes_across_dp_rank() - assert sizes is not None - is_sp = moe_state.is_sequence_parallel - group = get_ep_group() if is_sp else get_dp_group() - assert sizes[group.rank_in_group] == topk_ids.shape[0] - (topk_ids,) = group.all_gatherv([topk_ids], 0, sizes) - return topk_weights, topk_ids - - -def _hf_topk_router( - hidden_states: torch.Tensor, - gating_output: torch.Tensor, - topk: int, - renormalize: bool, - gate: nn.Module, -) -> tuple[torch.Tensor, torch.Tensor]: - """Route with a Transformers gate returning `(_, topk_weights, topk_ids)`.""" - _, topk_weights, topk_ids = gate(hidden_states) - return topk_weights, topk_ids - - -class TransformersRoutedExperts(RoutedExperts): - def get_expert_mapping( - self, - ckpt_gate_proj_name: str | None = None, - ckpt_down_proj_name: str | None = None, - ckpt_up_proj_name: str | None = None, - include_fused: bool = False, - ) -> list[tuple[str, str, int, str]]: - common_map = super().get_expert_mapping( - ckpt_gate_proj_name or "gate_proj", - ckpt_down_proj_name or "down_proj", - ckpt_up_proj_name or "up_proj", - include_fused, - ) - mixtral_map = super().get_expert_mapping("w1", "w2", "w3", include_fused) - if not include_fused: - return common_map + mixtral_map - common_fused, common_unfused = common_map[:3], common_map[3:] - mixtral_fused, mixtral_unfused = mixtral_map[:3], mixtral_map[3:] - return common_fused + mixtral_fused + common_unfused + mixtral_unfused - - -class MoEMixin(MixtureOfExperts, Base): - mlp_layers: list[nn.Module] - """MoE blocks whose experts were replaced, for the `MixtureOfExperts` methods.""" - - # Redeclared because `recursive_replace` assigns it below the first read - num_local_physical_experts: int - - def __init__(self, *, vllm_config: "VllmConfig", prefix: str = ""): - # Skip MixtureOfExperts.__init__ and call the next class in MRO - super(MixtureOfExperts, self).__init__(vllm_config=vllm_config, prefix=prefix) - - def update_physical_experts_metadata( - self, - num_physical_experts: int, - num_local_physical_experts: int, - ): - assert self.num_local_physical_experts == num_local_physical_experts - self.num_physical_experts = num_physical_experts - self.num_local_physical_experts = num_local_physical_experts - self.num_redundant_experts = num_physical_experts - self.num_logical_experts - for moe_block in self.mlp_layers: - moe_block.n_local_physical_experts = num_local_physical_experts - moe_block.n_physical_experts = num_physical_experts - moe_block.n_redundant_experts = self.num_redundant_experts - moe_block.experts.update_expert_map() - - def recursive_replace(self): - """Initialize the MoE layers.""" - experts_name = "experts" - text_config = self.text_config - - # Positional arguments - num_experts_names = CONFIG_FIELD_ALIASES["num_experts"] - num_experts = getattr_iter(text_config, num_experts_names, None) - top_k_names = CONFIG_FIELD_ALIASES["num_experts_per_token"] - top_k = getattr_iter(text_config, top_k_names, None) - assert top_k is not None - hidden_size = text_config.hidden_size - intermediate_size_names = CONFIG_FIELD_ALIASES["moe_intermediate_size"] - intermediate_size = getattr_iter(text_config, intermediate_size_names, None) - assert intermediate_size is not None - - num_shared_experts_names = CONFIG_FIELD_ALIASES["num_shared_experts"] - num_shared_experts = getattr_iter(text_config, num_shared_experts_names, 0) - - # Common kwargs - norm_topk_prob = getattr(text_config, "norm_topk_prob", None) - - # Routed scaling factor kwargs - routed_scaling_factor = getattr(text_config, "routed_scaling_factor", 1.0) - # aiter applies routed_scaling_factor internally - apply_routed_scale_to_output = not rocm_aiter_ops.is_fused_moe_enabled() - routed_scaling_factor_kwargs = dict( - routed_scaling_factor=routed_scaling_factor, - apply_routed_scale_to_output=apply_routed_scale_to_output, - ) - - # Dtype the router computes in, if it is not the activation dtype. - config_router_dtype = getattr(text_config, "moe_router_dtype", None) - if config_router_dtype is not None: - config_router_dtype = STR_DTYPE_TO_TORCH_DTYPE.get(config_router_dtype) - - # Grouped topk routing kwargs - num_expert_group = getattr(text_config, "n_group", None) - topk_group = getattr(text_config, "topk_group", None) - use_grouped_topk = num_expert_group is not None and topk_group is not None - grouped_topk_routing_kwargs = dict( - use_grouped_topk=use_grouped_topk, - num_expert_group=num_expert_group, - topk_group=topk_group, - ) - - # MoE activation function - activation = "silu" - wrapped_arch = self.config.architectures[0].lower() - if "gptoss" in wrapped_arch: - activation = "swigluoai" - - # Expert parallel load balancing kwargs - enable_eplb = self.parallel_config.enable_eplb - num_redundant_experts = self.parallel_config.eplb_config.num_redundant_experts - - # MixtureOfExperts mixin settings - ep_size = get_ep_group().world_size - - self.mlp_layers = [] - self.moe_layers = [] - self.num_expert_groups = 1 if num_expert_group is None else num_expert_group - self.num_logical_experts = num_experts - self.num_physical_experts = num_experts + num_redundant_experts - self.num_local_physical_experts = self.num_physical_experts // ep_size - self.num_routed_experts = num_experts - self.num_shared_experts = num_shared_experts - self.num_redundant_experts = num_redundant_experts - - # Down projections of shared experts consumed by FusedMoE - shared_down_projs: list[tuple[nn.Module, str]] = [] - - # Recursively fuse MoE layers - def _recursive_replace(module: nn.Module, prefix: str): - for child_name, child_module in module.named_children(): - qual_name = maybe_prefix(prefix, child_name) - # Naive implementations will have experts as ModuleList - is_modulelist = isinstance(child_module, nn.ModuleList) - # Packed implementations will have experts as 3D tensors of shapes like: - # gate_up_proj = (num_experts, 2 * intermediate_size, hidden_size) - # down_proj = (num_experts, intermediate_size, hidden_size) - params = list(child_module.parameters()) - is_3d = len(params) > 0 and all(p.ndim == 3 for p in params) - if child_name == experts_name and (is_modulelist or is_3d): - # Alias for readability - moe_block = module - experts = child_module - # Class of the fused block (parent of gate/experts/shared) - moe_block_cls = type(moe_block).__name__ - experts_cls = type(experts).__name__ - # Do the experts have biases - has_bias = False - for experts_param_name, _ in experts.named_parameters(): - if "bias" in experts_param_name: - has_bias = True - break - # Whether the expert weights are stored in transposed format. - # Comes from Transformers `use_experts_implementation` decorator. - is_transposed = getattr(experts, "is_transposed", False) - # If the config does not specify num_shared_experts, but - # the model has shared experts, we assume there is one. - if self.num_shared_experts == 0: - for moe_block_param_name, _ in moe_block.named_parameters(): - if "shared_expert" in moe_block_param_name: - self.num_shared_experts = 1 - break - - # Only pay for the layer index if something is per-layer; - # `extract_layer_index` raises for unnumbered module paths. - per_layer = (top_k, intermediate_size, norm_topk_prob) - layer_idx = ( - extract_layer_index(qual_name) - if any(isinstance(v, list) for v in per_layer) - else 0 - ) - layer_top_k = maybe_per_layer(top_k, layer_idx) - layer_renormalize = maybe_per_layer(norm_topk_prob, layer_idx) - if layer_renormalize is None: - layer_renormalize = layer_top_k > 1 - - kwargs: dict[str, Any] = dict( - num_experts=num_experts, - top_k=layer_top_k, - hidden_size=hidden_size, - intermediate_size=maybe_per_layer(intermediate_size, layer_idx), - renormalize=layer_renormalize, - quant_config=self.quant_config, - prefix=qual_name, - activation=activation, - enable_eplb=enable_eplb, - num_redundant_experts=num_redundant_experts, - has_bias=has_bias, - is_fused_checkpoint_transposed=is_transposed, - routed_experts_cls=TransformersRoutedExperts, - ) - # _maybe_apply_routed_scale_to_output edge case. Transformers - # decoder layers do not compensate for dividing by scaling factor. - divides_shared_output = ( - routed_scaling_factor != 1.0 - and apply_routed_scale_to_output - and self.model_config.dtype == torch.float16 - ) - fuser = MoEBlockFuser.match( - moe_block, experts_name, divides_shared_output - ) - if fuser is not None: - # MoE block forward is fully replaced. - # gate/router and shared expert (if any) runs in MoERunner. - shared_experts = fuser.shared_experts(moe_block, prefix) - # Store shared experts for later down projection adjustment - if shared_experts is not None: - hf_shared = shared_experts.shared_experts - glu_fuser = get_fuser(hf_shared, GLUFuser) - if glu_fuser is not None: - down_name = glu_fuser.down_name - if down_name is not None: - shared_down_projs.append((hf_shared, down_name)) - if scoring_func := fuser.scoring_func: - # Internal routing kwargs. - # Prefer config, otherwise read it from fuser. - router_dtype = config_router_dtype or fuser.router_dtype - gate = fuser.gate(moe_block, prefix, router_dtype) - kwargs |= dict( - scoring_func=scoring_func, - renormalize=( - layer_renormalize - if fuser.renormalize is None - else fuser.renormalize - ), - gate=gate, - ) - if router_dtype is not None: - kwargs["router_logits_dtype"] = router_dtype - if use_grouped_topk: - kwargs |= grouped_topk_routing_kwargs - if routed_scaling_factor != 1.0: - kwargs |= routed_scaling_factor_kwargs - bias = getattr(gate, "e_score_correction_bias", None) - if bias is not None: - kwargs["e_score_correction_bias"] = bias - else: - # External routing kwargs - kwargs |= dict( - num_expert_group=num_expert_group, - topk_group=topk_group, - custom_routing_function=partial( - _hf_topk_router, - gate=getattr(moe_block, fuser.gate_name), - ), - ) - - kwargs |= dict( - is_sequence_parallel=( - self.parallel_config.use_sequence_parallel_moe - ), - shared_experts=shared_experts, - ) - fuser.rewrite_forward(moe_block) - logger.info_once( - "Fused: gate + experts%s (%s) -> MoERunner (%s routing)", - " + shared experts" if fuser.shared_name else "", - moe_block_cls, - "external" if fuser.scoring_func is None else "internal", - ) - else: - # MoE block forward is unmodified. - # gate/router and shared expert (if any) runs in Transformers. - # We then smuggle the topk_ids in using a custom op. - moe_state = TransformersMoEState() - - kwargs |= dict( - num_expert_group=num_expert_group, - topk_group=topk_group, - custom_routing_function=partial( - _hf_topk_router_precomputed, moe_state=moe_state - ), - runner_cls=TransformersMoERunner, - runner_args={"moe_state": moe_state}, - ) - logger.info_once( - "Fused: experts (%s) -> MoERunner (external routing)", - experts_cls, - ) - fused_experts = FusedMoEFactory(**kwargs) - moe_block.experts = fused_experts - log_replacement(qual_name, experts, fused_experts) - # Update MixtureOfExperts mixin state - self.mlp_layers.append(moe_block) - self.moe_layers.append(fused_experts) - else: - _recursive_replace(child_module, prefix=qual_name) - - _recursive_replace(self.model, prefix="model") - self.num_moe_layers = len(self.moe_layers) - # Continue with the replacement of layers in Base - super().recursive_replace() - # GLUFuser likely fused shared_experts. The down projection after the GLU - # normally immediately reduces but we want FusedMoE to handle the reduction. - for hf_shared, down_name in shared_down_projs: - hf_shared.get_submodule(down_name).reduce_results = False diff --git a/vllm/model_executor/models/transformers/multimodal.py b/vllm/model_executor/models/transformers/multimodal.py deleted file mode 100644 index 12771d2050..0000000000 --- a/vllm/model_executor/models/transformers/multimodal.py +++ /dev/null @@ -1,1267 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -# Copyright 2024 The vLLM team. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Transformers modeling backend mixin for multi-modal models.""" - -import math -from collections import defaultdict -from collections.abc import Callable, Iterable, Mapping, Sequence -from contextlib import ExitStack, contextmanager -from functools import cached_property -from typing import TYPE_CHECKING, Any - -import torch -import transformers - -from vllm.compilation.decorators import should_torch_compile_mm_encoder -from vllm.config.utils import getattr_iter -from vllm.inputs import MultiModalDataBuiltins, MultiModalDataDict -from vllm.logger import init_logger -from vllm.model_executor.models.interfaces import ( - MultiModalEmbeddings, - SupportsMRoPE, - SupportsMultiModal, -) -from vllm.model_executor.models.module_mapping import MultiModelKeys -from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargsItems -from vllm.multimodal.inputs import MultiModalFeatureSpec, MultiModalFieldConfig -from vllm.multimodal.parse import ImageSize, MultiModalDataItems, MultiModalDataParser -from vllm.multimodal.processing import ( - BaseDummyInputsBuilder, - BaseMultiModalProcessor, - BaseProcessingInfo, - PromptReplacement, - PromptUpdate, - PromptUpdateDetails, - cached_encode, -) -from vllm.multimodal.processing.processor import HFMultiModalInputs -from vllm.sequence import IntermediateTensors -from vllm.utils.gpu_sync_debug import gpu_sync_allowed -from vllm.utils.torch_utils import async_tensor_h2d - -from .base import Base - -if TYPE_CHECKING: - from transformers import BatchFeature, PreTrainedModel - - from vllm.config import VllmConfig - from vllm.config.multimodal import MultiModalDummyOptions - -logger = init_logger(__name__) - -_MODALITY_TO_TOKEN_TYPE_ID = {"image": 1, "video": 2, "audio": 3} -_MODALITY_SIZE_KEYS = { - "audio": "num_audio_tokens", - "image": "num_image_patches", - "video": "num_video_patches", -} -# NOTE: Profiling cap as in llava_onevision._MAX_FRAMES_PER_VIDEO -# past which a pixel budget only shrinks the frames, so Qwen3-VL's most is -# 12168 tokens at 16 frames and 9600 at its `max_frames` of 768 -_MAX_FRAMES_PER_VIDEO = 16 -# Arbitrary large size, bounded only by the processor's own resizing -_MAX_DUMMY_SIDE = 10_000 -# Tiny inputs trip processors' channel-axis inference -_MIN_DUMMY_SIDE = 224 - - -def _get_embed_token_id(replacement_ids: torch.Tensor) -> int: - """The token an expansion repeats is the one holding the embeddings.""" - return int(replacement_ids.mode().values) - - -def _count_embed_tokens(seqs: list[list[int]]) -> torch.Tensor: - """Number of embedding tokens in each item's replacement.""" - counts = [] - for seq in seqs: - ids = torch.tensor(seq) - counts.append(int(ids.eq(_get_embed_token_id(ids)).sum())) - return torch.tensor(counts) - - -class MultiModalProcessingInfo(BaseProcessingInfo): - def _get_audio_processor(self) -> Any: - # TODO: drop feature_extractor branch once huggingface/transformers#44394 lands. - return getattr_iter( - self.get_hf_processor(), ("audio_processor", "feature_extractor") - ) - - @cached_property - def _is_audio_model(self) -> bool: - return self._get_audio_processor() is not None - - @cached_property - def _is_image_model(self) -> bool: - return hasattr(self.get_hf_processor(), "image_processor") - - @cached_property - def _is_video_model(self) -> bool: - if not hasattr(self.get_hf_processor(), "video_processor"): - return False - try: - Base.check_version("5.18.0", "video inputs") - except ImportError as e: - logger.info_once("%s, so video inputs are disabled.", e) - return False - num_frames = self._get_min_video_frames() - side = _MIN_DUMMY_SIDE - # TODO: Drop the except branch once every video processor can count, see - # https://github.com/huggingface/transformers/issues/43329 - try: - mm_tokens = self._get_num_mm_tokens(video_sizes=([num_frames, side, side],)) - except AttributeError: - logger.info_once( - "%s cannot count video tokens yet, so the Transformers modeling " - "backend serves this model without video inputs. See " - "https://github.com/huggingface/transformers/issues/43329", - type(self.get_hf_processor()).__name__, - ) - return False - return mm_tokens["num_video_tokens"] is not None - - @cached_property - def _video_needs_metadata(self) -> bool: - video_processor = getattr(self.get_hf_processor(), "video_processor", None) - return getattr(video_processor, "do_sample_frames", False) - - def _get_min_video_frames(self) -> int: - video_processor = self.get_hf_processor().video_processor - # A processor without `temporal_patch_size` groups no frames, so one is enough - return getattr(video_processor, "temporal_patch_size", 1) - - def _get_supported_modalities(self) -> list[str]: - modalities = [] - if self._is_audio_model: - modalities.append("audio") - if self._is_image_model: - modalities.append("image") - if self._is_video_model: - modalities.append("video") - if not modalities: - raise ValueError( - f"{type(self.get_hf_processor()).__name__} exposes no image, video " - "or audio processor, so the Transformers modeling backend cannot " - "serve this model as multi-modal." - ) - return modalities - - def _get_audio_sampling_rate(self) -> float: - sub = self._get_audio_processor() - if sub is not None and hasattr(sub, "sampling_rate"): - return sub.sampling_rate - return 16000.0 - - def get_data_parser(self) -> MultiModalDataParser: - target_sr = self._get_audio_sampling_rate() if self._is_audio_model else None - return MultiModalDataParser( - target_sr=target_sr, - video_needs_metadata=self._video_needs_metadata, - expected_hidden_size=self._get_expected_hidden_size(), - allow_missing_mm_embeddings=self.allow_missing_mm_embeddings, - ) - - def get_supported_mm_limits(self): - return dict.fromkeys(self._get_supported_modalities()) - - def get_mm_max_tokens_per_item(self, seq_len, mm_counts): - modalities = self._get_supported_modalities() - max_tokens = {} - if "audio" in modalities: - max_tokens["audio"] = self.get_max_audio_tokens() - if "image" in modalities: - max_tokens["image"] = self.get_max_image_tokens() - if "video" in modalities: - max_tokens["video"] = self.get_max_video_tokens(seq_len, mm_counts) - return max_tokens - - def get_max_audio_tokens(self) -> int: - config = self.get_hf_config() - audio_config_names = ("audio_config", "encoder_config") - names = ("max_source_positions", "max_position_embeddings", "max_pos_emb") - audio_config = getattr_iter(config, audio_config_names, default=config) - val = getattr_iter(audio_config, names) - if val is not None: - return int(val) - raise ValueError( - f"Unable to get max input length from {type(audio_config).__name__}. " - f"The following attribute names were checked: {names}." - ) - - def _get_num_mm_tokens(self, **sizes: Sequence[Sequence[int]]) -> Any: - processor = self.get_hf_processor() - multimodal_config = self.ctx.model_config.get_multimodal_config() - mm_processor_kwargs = multimodal_config.mm_processor_kwargs or {} - return processor._get_num_multimodal_tokens(**sizes, **mm_processor_kwargs) - - def get_max_image_tokens(self) -> int: - size = self.get_image_size_with_most_features() - return self._get_num_image_tokens(size) - - def get_max_video_tokens(self, seq_len: int, mm_counts: Mapping[str, int]) -> int: - num_frames = self.get_num_frames_with_most_features(seq_len, mm_counts) - return self._get_max_video_tokens_for_frames(num_frames) - - def _get_max_video_tokens_for_frames(self, num_frames: int) -> int: - size = self.get_video_size_with_most_features(num_frames) - return self._get_num_video_tokens(num_frames, size) - - def _get_num_image_tokens(self, size: ImageSize) -> int: - mm_tokens = self._get_num_mm_tokens(image_sizes=([size.height, size.width],)) - return mm_tokens["num_image_tokens"][0] - - def _get_num_video_tokens(self, num_frames: int, size: ImageSize) -> int: - video_sizes = ([num_frames, size.height, size.width],) - mm_tokens = self._get_num_mm_tokens(video_sizes=video_sizes) - return mm_tokens["num_video_tokens"][0] - - def _get_size_candidates( - self, sub_processor: Any, divisors: tuple[int, ...] - ) -> list[ImageSize]: - """Candidate sizes read off a sub-processor's `size`. - - The keys are one of `VALID_SIZE_DICT_KEYS`, so the bound is either exact - or an area budget, which `divisors` splits over its items. `shortest_edge` - bounds only the small side, so it yields no candidate. - - `longest_edge` is read as the area budget Qwen and GLM use it for. A - processor using it as an edge length (SmolVLM) yields a tiny candidate, - which is only chosen if it ties on tokens, so it is still correct. - """ - size = getattr(sub_processor, "size", None) or {} - height = size.get("height", size.get("max_height")) - width = size.get("width", size.get("max_width")) - if height and width: - return [ImageSize(width=width, height=height)] - if max_pixels := size.get("max_pixels", size.get("longest_edge")): - sides = {math.isqrt(max_pixels // divisor) for divisor in divisors} - return [ - ImageSize(width=side, height=side) for side in sorted(sides) if side - ] - return [] - - def _get_size_with_most_tokens( - self, candidates: list[ImageSize], count: Callable[[ImageSize], int] - ) -> ImageSize: - """Smallest size found that yields the most tokens. - - `size` bounds the resized output, not the token count, so the candidates - are compared by token count against an arbitrary large fallback, which - tiling processors need. The fallback goes last so a candidate wins a tie. - Resize rounding can give a large input fewer tokens (Qwen2.5-VL), so the - candidates matter for the count, not only for memory. - - The winner is then halved while the count holds, so processors with no - usable bound aren't profiled on huge dummy inputs. - """ - sizes = [*candidates, ImageSize(width=_MAX_DUMMY_SIDE, height=_MAX_DUMMY_SIDE)] - num_tokens = [count(size) for size in sizes] - most_tokens = max(num_tokens) - size = sizes[num_tokens.index(most_tokens)] - - while min(size.width, size.height) // 2 >= _MIN_DUMMY_SIDE: - smaller = ImageSize(width=size.width // 2, height=size.height // 2) - if count(smaller) < most_tokens: - break - size = smaller - return size - - def get_image_size_with_most_features(self) -> ImageSize: - return self._get_size_with_most_tokens( - self._get_size_candidates(self.get_hf_processor().image_processor, (1,)), - self._get_num_image_tokens, - ) - - def get_video_size_with_most_features(self, num_frames: int) -> ImageSize: - """Per-frame size that yields the most video tokens. - - The video processor's `size` budget may cover one frame, every frame or - every temporal patch, so all three splits are tried. - """ - video_processor = self.get_hf_processor().video_processor - grid_t = max(num_frames // self._get_min_video_frames(), 1) - return self._get_size_with_most_tokens( - self._get_size_candidates(video_processor, (num_frames, grid_t, 1)), - lambda size: self._get_num_video_tokens(num_frames, size), - ) - - def get_num_frames_with_most_features( - self, seq_len: int, mm_counts: Mapping[str, int] - ) -> int: - max_videos = max(mm_counts.get("video", 0), 1) - step = self._get_min_video_frames() - num_frames = step - while ( - num_frames + step <= _MAX_FRAMES_PER_VIDEO - and self._get_max_video_tokens_for_frames(num_frames + step) * max_videos - <= seq_len - ): - num_frames += step - return num_frames - - -class MultiModalDummyInputsBuilder(BaseDummyInputsBuilder[MultiModalProcessingInfo]): - def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str: - text = "" - if self.info._is_audio_model and (num_audios := mm_counts.get("audio", 0)): - processor = self.info.get_hf_processor() - audio_token = getattr(processor, "audio_token", "") - # Separated so that adjacent placeholders stay distinguishable - text += " ".join([audio_token] * num_audios) - if self.info._is_image_model and (num_images := mm_counts.get("image", 0)): - processor = self.info.get_hf_processor() - if "gemma3" in processor.__class__.__name__.lower(): - image_token = processor.boi_token - else: - image_token = getattr(processor, "image_token", "") - # Some processors (e.g. HunYuanVL) reject a bare image token and - # require each one to be wrapped in its start/end markers. - start_token = getattr(processor, "image_start_token", "") - end_token = getattr(processor, "image_end_token", "") - image_token = f"{start_token}{image_token}{end_token}" - text += image_token * num_images - if self.info._is_video_model and (num_videos := mm_counts.get("video", 0)): - processor = self.info.get_hf_processor() - video_token = getattr(processor, "video_token", "") - text += video_token * num_videos - return text - - def get_dummy_mm_data( - self, - seq_len: int, - mm_counts: Mapping[str, int], - mm_options: "MultiModalDummyOptions", - ) -> MultiModalDataDict: - data = MultiModalDataBuiltins() - if self.info._is_audio_model and (num_audios := mm_counts.get("audio", 0)): - sampling_rate = self.info._get_audio_sampling_rate() - sub = self.info._get_audio_processor() - chunk_length = getattr(sub, "chunk_length", None) if sub else None - if chunk_length is None: - chunk_length = 30 - data["audio"] = self._get_dummy_audios( - length=int(chunk_length * sampling_rate), - num_audios=num_audios, - overrides=mm_options.get("audio"), - ) - if self.info._is_image_model and (num_images := mm_counts.get("image", 0)): - width, height = self.info.get_image_size_with_most_features() - data["image"] = self._get_dummy_images( - width=width, - height=height, - num_images=num_images, - overrides=mm_options.get("image"), - ) - if self.info._is_video_model and (num_videos := mm_counts.get("video", 0)): - num_frames = self.info.get_num_frames_with_most_features(seq_len, mm_counts) - width, height = self.info.get_video_size_with_most_features(num_frames) - videos = self._get_dummy_videos( - width=width, - height=height, - num_frames=num_frames, - num_videos=num_videos, - overrides=mm_options.get("video"), - ) - # Only processors that sample frames need metadata, which has the - # dummy frames consumed verbatim - if self.info._video_needs_metadata: - video_processor = self.info.get_hf_processor().video_processor - fps = getattr(video_processor, "fps", None) - videos = [ - ( - video, - { - "fps": fps, - "duration": len(video) / fps if fps else None, - "total_num_frames": len(video), - "frames_indices": list(range(len(video))), - "video_backend": "opencv", - "do_sample_frames": False, - }, - ) - for video in videos - ] - data["video"] = videos - return data - - -class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]): - """Locates placeholders from the `text_replacement_offsets` the HF processor - reports, expressing each one as a `PromptUpdate`. - - Stating the expansion as an update is what lets it be rebuilt from an - unexpanded prompt, so this processor takes the base class's processing path - and with it the multi-modal processor cache. - """ - - def _get_hf_mm_inputs( - self, - mm_items: MultiModalDataItems, - hf_kwargs: Mapping[str, object], - ) -> HFMultiModalInputs: - """Pass the video metadata along to the HF processor.""" - hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs) - hf_data = hf_inputs.hf_data - - if "videos" not in hf_data or not self.info._video_needs_metadata: - return hf_inputs - - videos = hf_data["videos"] - assert isinstance(videos, Sequence) - videos, metadata = zip(*videos) - hf_data["videos"] = list(videos) - hf_data["video_metadata"] = [ - {k: v for k, v in item.items() if k != "do_sample_frames"} - for item in metadata - ] - do_sample_frames = {item.get("do_sample_frames", False) for item in metadata} - if len(do_sample_frames) > 1: - raise ValueError( - "The videos in a request must agree on `do_sample_frames`, " - "since the HF processor takes one value for all of them." - ) - - return hf_inputs._replace( - hf_kwargs={ - "do_sample_frames": do_sample_frames.pop(), - **hf_inputs.hf_kwargs, - } - ) - - def _get_modality_field_names(self, modality: str) -> set[str]: - """Names of the fields the sub-processor for `modality` produces.""" - # TODO: use else branch only once huggingface/transformers#44394 lands. - if modality == "audio": - sub_processor = self.info._get_audio_processor() - else: - processor = self.info.get_hf_processor() - sub_processor = getattr(processor, f"{modality}_processor", None) - - # Pre-computed embeddings bypass the sub-processor entirely - names = {f"{modality}_embeds"} - for name in getattr(sub_processor, "model_input_names", None) or (): - # Companion masks are emitted but not always declared - names.update((name, f"{name}_mask")) - return names - - def _partition_keys_by_modality( - self, - keys: list[str], - modalities: list[str], - ) -> dict[str, list[str]]: - """Attribute each HF processor output key to the modality that produced it.""" - if len(modalities) == 1: - return {modalities[0]: keys} - - claimed = {m: self._get_modality_field_names(m) for m in modalities} - - owned: dict[str, list[str]] = {modality: [] for modality in modalities} - unclaimed = [] - for key in keys: - for modality in modalities: - if key in claimed[modality]: - owned[modality].append(key) - break - else: - unclaimed.append(key) - - if unclaimed: - logger.warning_once( - "Unable to attribute %s to any of the modalities %s, so they " - "will not be passed to the model. Add them to the relevant " - "sub-processor's `model_input_names` to fix this.", - tuple(unclaimed), - tuple(modalities), - ) - - return owned - - def _get_slice_dim(self, data: Any, total_rows: int) -> int: - """Which dimension of a field holds the rows belonging to each item. - - Some processors (e.g., Idefics3) return image fields with a leading batch - dimension, putting the rows one dimension further in. - """ - if not isinstance(data, torch.Tensor) or data.ndim < 2: - return 0 - if data.shape[0] != total_rows and data.shape[1] == total_rows: - return 1 - return 0 - - def _get_mm_fields_config( - self, - hf_inputs: "BatchFeature", - hf_processor_mm_kwargs: Mapping[str, object], - ) -> Mapping[str, MultiModalFieldConfig]: - # HF Processors always return a mask but vLLM doesn't need it - hf_inputs.pop("attention_mask", None) - - # Absent if the modality had no items - sizes = { - modality: hf_inputs.get(key) - for modality, key in _MODALITY_SIZE_KEYS.items() - } - modalities = [m for m, size in sizes.items() if size is not None] - - # Keys we wrote ourselves, rather than ones a sub-processor produced - own_keys = {*_MODALITY_SIZE_KEYS.values(), "num_video_tokens"} | { - f"{modality}_replacement_{suffix}" - for modality in modalities - for suffix in ("ids", "sizes") - } - # Registered by name below, so no sub-processor has to claim them - own_keys |= {"image_grid_thw", "video_grid_thw", "second_per_grid_ts"} - keys = [key for key in hf_inputs if key not in own_keys] - owned = self._partition_keys_by_modality(keys, modalities) - - mm_fields = { - key: self._get_field_config(modality, key, hf_inputs, sizes[modality]) - for modality in modalities - for key in owned[modality] - } - - for modality in modalities: - # One row per item, and only ever read on the CPU - mm_fields[_MODALITY_SIZE_KEYS[modality]] = MultiModalFieldConfig.batched( - modality, keep_on_cpu=True - ) - replacement_sizes = hf_inputs.get(f"{modality}_replacement_sizes") - if replacement_sizes is not None: - mm_fields[f"{modality}_replacement_ids"] = ( - MultiModalFieldConfig.flat_from_sizes( - modality, replacement_sizes, keep_on_cpu=True - ) - ) - - if "image" in modalities: - # Always one row per item, whatever they describe - mm_fields["image_grid_thw"] = MultiModalFieldConfig.batched( - "image", keep_on_cpu=True - ) - if "video" in modalities: - mm_fields["video_grid_thw"] = MultiModalFieldConfig.batched( - "video", keep_on_cpu=True - ) - mm_fields["second_per_grid_ts"] = MultiModalFieldConfig.batched( - "video", keep_on_cpu=True - ) - - return mm_fields - - def _get_field_config( - self, - modality: str, - key: str, - hf_inputs: "BatchFeature", - sizes: torch.Tensor, - ) -> MultiModalFieldConfig: - data = hf_inputs[key] - # Un-padded fields are already one entry per item, so index rather than slice - if modality == "audio" or isinstance(data, list): - return MultiModalFieldConfig.batched(modality) - - total = int(sizes.sum()) - dim = self._get_slice_dim(data, total) - if modality != "video": - return MultiModalFieldConfig.flat_from_sizes(modality, sizes, dim=dim) - - rows = data.shape[dim] - if rows == len(sizes): - return MultiModalFieldConfig.batched("video") - if rows == total: - return MultiModalFieldConfig.flat_from_sizes("video", sizes, dim=dim) - # VideoLLaMA3's compression mask has one row per token the processor counts - num_video_tokens = hf_inputs["num_video_tokens"] - if rows == int(num_video_tokens.sum()): - return MultiModalFieldConfig.flat_from_sizes( - "video", num_video_tokens, dim=dim - ) - # NOTE: Any other layout would need a per-model guess, which we'd like to avoid - raise ValueError( - f"{type(self.info.get_hf_processor()).__name__} returned {rows} row(s) of " - f"`{key}` for {len(sizes)} video(s) with {total} patch(es), so the rows " - "cannot be attributed to a video." - ) - - def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str: - return self.dummy_inputs.get_dummy_text(mm_counts) - - def _unpad_audios( - self, - hf_inputs: "BatchFeature", - mm_data: Mapping[str, object], - mm_kwargs: Mapping[str, object], - ) -> None: - """Replace the audio fields with each audio processed on its own. - - Processors pad every audio up to the longest in the call, which would leave - an item's data dependent on what it was processed with. Unlike images, - nothing in the output states how long each one really is, and processors - pad a lone audio too, so the only way to know what an audio produces by - itself is to process it by itself. - """ - audios = mm_data.get("audio") - if not isinstance(audios, Iterable) or not audios: - return - if len({len(audio) for audio in audios}) == 1: - return - - alone = [ - self.info.ctx.call_hf_processor( - self.info.get_hf_processor(**mm_kwargs), - dict( - text=self.dummy_inputs.get_dummy_text({"audio": 1}), audio=[audio] - ), - mm_kwargs, - ) - for audio in audios - ] - - for key in self._get_modality_field_names("audio"): - if isinstance(hf_inputs.get(key), torch.Tensor): - hf_inputs[key] = [output[key][0] for output in alone] - - def _unpad_images(self, hf_inputs: "BatchFeature") -> None: - """Trim each image back to its own size when the processor padded them all - to the largest in the batch. - - An image's data has to depend on nothing but that image, or the multi-modal - processor cache would store it under that image's hash and later reuse it - beside a different neighbour. Padding is re-applied when the encoder runs. - """ - pixel_values = hf_inputs.get("pixel_values") - image_sizes = hf_inputs.get("image_sizes") - if not isinstance(pixel_values, torch.Tensor): - return - if not isinstance(image_sizes, torch.Tensor): - return - if pixel_values.ndim != 4 or len(pixel_values) != len(image_sizes): - return - - # The sizes describe the trailing dimensions only if the largest of them is - # what the batch was padded up to. Otherwise they mean something else, as - # in llava-onevision, where they are the sizes before any processing. - maxima = image_sizes.max(dim=0).values - if maxima.tolist() != list(pixel_values.shape[-2:]): - return - - hf_inputs["pixel_values"] = [ - image[..., :height, :width] - for image, (height, width) in zip(pixel_values, image_sizes.tolist()) - ] - - def _get_prompt_updates( - self, - mm_items: MultiModalDataItems, - hf_processor_mm_kwargs: Mapping[str, object], - out_mm_kwargs: MultiModalKwargsItems, - ) -> Sequence[PromptUpdate]: - """Replace each modality's placeholder token with the token ids that item's - replacement text encodes to, marking which of them hold embeddings.""" - hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs) - tokenizer = self.info.get_tokenizer() - - def get_target_token_ids(token: str | int | Sequence[int]) -> list[int]: - if isinstance(token, str): - return cached_encode(tokenizer, token, add_special_tokens=False) - if isinstance(token, int): - return [token] - return list(token) - - updates = [] - for modality, items in out_mm_kwargs.items(): - # Popped so they are neither cached nor sent to the model; the updates - # they produce are cached alongside the item instead - replacements = [] - for item in items: - ids = item.pop(f"{modality}_replacement_ids").data - assert isinstance(ids, torch.Tensor) - replacements.append( - PromptUpdateDetails.select_token_id( - ids.tolist(), _get_embed_token_id(ids) - ) - ) - token = getattr(hf_processor, f"{modality}_token") - updates.append( - PromptReplacement( - modality=modality, - target=get_target_token_ids(token), - replacement=replacements.__getitem__, - ) - ) - return updates - - def _get_num_image_patches( - self, - hf_inputs: "BatchFeature", - mm_data: Mapping[str, object], - num_images: int, - ) -> torch.Tensor: - """How many rows of the image fields belong to each image. - - Taken from whichever per-image count the processor reports, - and checked against the data it has to slice. - """ - if (grid := hf_inputs.get("image_grid_thw")) is not None: - num_patches = grid.prod(-1) - elif (counts := self._get_num_patches_per_image(mm_data)) is not None: - num_patches = torch.tensor(counts) - else: - num_patches = torch.ones(num_images, dtype=torch.long) - - image_data = hf_inputs.get("pixel_values", hf_inputs.get("image_patches")) - if isinstance(image_data, torch.Tensor): - total = int(num_patches.sum()) - rows = image_data.shape[self._get_slice_dim(image_data, total)] - if rows != total: - raise ValueError( - f"{type(self.info.get_hf_processor()).__name__} returned " - f"{rows} row(s) of image data for {num_images} image(s), which " - f"cannot be split into the {num_patches.tolist()} row(s) per " - "image derived from its outputs, so the rows cannot be " - "attributed to an image. Gemma3 does this when " - "`do_pan_and_scan` crops an image." - ) - return num_patches - - def _get_num_patches_per_image( - self, mm_data: Mapping[str, object] - ) -> list[int] | None: - """Ask the HF processor how many rows of image data each image produces.""" - images = mm_data.get("images") - if not isinstance(images, Iterable) or not images: - return None - try: - sizes = [(image.height, image.width) for image in images] - mm_tokens = self.info.get_hf_processor()._get_num_multimodal_tokens( - image_sizes=sizes, **self.info.ctx.get_merged_mm_kwargs({}) - ) - return list(mm_tokens["num_image_patches"]) - except (AttributeError, KeyError, TypeError): - return None - - def _apply_hf_processor_main( - self, - mm_items: MultiModalDataItems, - hf_kwargs: Mapping[str, object], - ) -> "BatchFeature": - hf_data, hf_kwargs, passthrough_data = self._get_hf_mm_inputs( - mm_items, hf_kwargs - ) - - if not hf_data: - return self._finalize_hf_mm_data(hf_data, hf_kwargs, passthrough_data) - - prompt_text = hf_data.pop("text") - assert isinstance(prompt_text, str) - - # Ask for the replacement each placeholder expands to, and record it as - # per-item fields: its token ids, and the tokens or patches behind them - if has_mm_data := any(hf_data.values()): - hf_data = {**hf_data, "return_text_replacement_offsets": True} - - try: - hf_inputs = self.info.ctx.call_hf_processor( - self.info.get_hf_processor(**hf_kwargs), - dict(text=prompt_text, **hf_data), - hf_kwargs, - ) - except ValueError: - if any(hf_data.values()): - raise - # Some processors reject a prompt holding placeholders with - # no data to go with them, so tokenize it without them - tokenizer = self.info.get_tokenizer() - hf_inputs = transformers.BatchFeature( - dict(input_ids=[tokenizer.encode(prompt_text)]), tensor_type="pt" - ) - self._unpad_images(hf_inputs) - self._unpad_audios(hf_inputs, hf_data, hf_kwargs) - - # Drop the inputs the model would reject - hf_inputs.pop("mm_token_type_ids", None) - hf_inputs.pop("token_type_ids", None) - - offsets = hf_inputs.pop("text_replacement_offsets", None) - # Some processors return an empty batch as a tensor rather than a list - if offsets is None or len(offsets) == 0 or len(offsets[0]) == 0: - if has_mm_data: - raise ValueError( - f"{type(self.info.get_hf_processor()).__name__} returned no " - "text replacement offsets, so the Transformers modeling backend " - "cannot locate the placeholder of each item. Its `__call__` has " - "to reach `ProcessorMixin.get_text_with_replacements` with one " - "replacement per item, which usually means implementing " - "`replace__token`. Please report this to transformers " - "so it can be fixed." - ) - hf_inputs.pop("input_ids", None) - return self._finalize_hf_mm_data( - hf_data, hf_kwargs, passthrough_data, hf_inputs - ) - - tokenizer = self.info.get_tokenizer() - replacements = defaultdict[str, list[list[int]]](list) - for entry in offsets[0]: - replacements[entry["type"]].append( - cached_encode(tokenizer, entry["replacement"], add_special_tokens=False) - ) - - for modality, seqs in replacements.items(): - hf_inputs[f"{modality}_replacement_ids"] = torch.tensor( - [token_id for seq in seqs for token_id in seq] - ) - hf_inputs[f"{modality}_replacement_sizes"] = torch.tensor( - [len(seq) for seq in seqs] - ) - if modality == "image": - hf_inputs["num_image_patches"] = self._get_num_image_patches( - hf_inputs, hf_data, len(seqs) - ) - elif modality == "audio": - hf_inputs["num_audio_tokens"] = _count_embed_tokens(seqs) - elif modality == "video": - grid = hf_inputs.get("video_grid_thw") - hf_inputs["num_video_patches"] = ( - grid.prod(-1) - if grid is not None - else torch.ones(len(seqs), dtype=torch.long) - ) - hf_inputs["num_video_tokens"] = _count_embed_tokens(seqs) - - hf_inputs.pop("input_ids") - - return self._finalize_hf_mm_data( - hf_data, hf_kwargs, passthrough_data, hf_inputs - ) - - -class MultiModalMixin(SupportsMultiModal, SupportsMRoPE, Base): - def __init__(self, *, vllm_config: "VllmConfig", prefix: str = ""): - # Skip SupportsMRoPE.__init__ and call the next class in MRO - super(SupportsMRoPE, self).__init__(vllm_config=vllm_config, prefix=prefix) - - def _find_encoder_classes( - self, model: "PreTrainedModel" - ) -> dict[str, type["PreTrainedModel"]]: - """Modalities whose encoder cannot be told apart from the model itself are - omitted, as are those `get_encoder` rejects.""" - encoder_classes: dict[str, type[PreTrainedModel]] = {} - for modality in _MODALITY_TO_TOKEN_TYPE_ID: - try: - encoder_cls = type(model.get_encoder(modality=modality)) - except (TypeError, ValueError): - continue - if encoder_cls is not type(model): - encoder_classes[modality] = encoder_cls - return encoder_classes - - @contextmanager - def _mark_model_components(self, vllm_config: "VllmConfig"): - model_config = vllm_config.model_config - encoder_classes = self._pre_trained_model_classes.encoders - if not encoder_classes: - logger.debug("No encoders identified, so no components will be marked") - yield - return - - if model_config.skip_tokenizer_init: - # Determining the supported modalities needs the HF processor, which in - # turn needs a tokenizer - mm_config = model_config.get_multimodal_config() - if mm_config.mm_encoder_only or any( - mm_config.get_limit_per_prompt(modality) == 0 - for modality in encoder_classes - ): - logger.warning_once( - "Unable to determine the supported modalities without a " - "tokenizer, so no model components will be skipped." - ) - yield - return - - # Modalities we don't serve report a limit of 999, which would stop their - # encoder ever being skipped - supported_modalities = MULTIMODAL_REGISTRY.get_processing_info( - model_config - ).supported_mm_limits - - # One encoder often serves several modalities, and may only be skipped when - # all of them are disabled, so mark it once for the whole set - modalities_by_encoder = defaultdict(set) - for modality, encoder_cls in encoder_classes.items(): - if modality in supported_modalities: - modalities_by_encoder[encoder_cls].add(modality) - - with ExitStack() as stack: - stack.enter_context( - self._mark_language_model( - vllm_config, targets=self._pre_trained_model_classes.decoder - ) - ) - for encoder_cls, modalities in modalities_by_encoder.items(): - stack.enter_context( - self._mark_tower_model(vllm_config, modalities, targets=encoder_cls) - ) - yield - - def _decorate_for_torch_compile(self): - """Decorate the model's decoder and encoder classes to indicate to vLLM - that they support torch compile if `can_enable_torch_compile` and - `should_torch_compile_mm_encoder` are True respectively. - """ - super()._decorate_for_torch_compile() - # Decorate the encoder model classes to support torch compile if needed - if self.compilation_config.compile_mm_encoder: - encoder_classes = self._pre_trained_model_classes.encoders - if not encoder_classes: - raise ValueError( - "Unable to infer any encoder classes from the model. " - "You must either: update the model so that " - "https://huggingface.co/docs/transformers/en/main_classes/model#transformers.PreTrainedModel.get_encoder" - " can detect the encoders correctly, or remove " - "'compile_mm_encoder'." - ) - logger.warning_once( - "Multimodal encoder compilation with the Transformers modeling backend " - "is an experimental feature. It relies on:\n" - "- The encoder being torch compilable.\n" - "- All encoder tensor inputs must be type hinted as either " - "`torch.Tensor` or `torch.FloatTensor`.\n" - "- The 0-th dimension of all tensor inputs to the encoder being the " - "dynamic dimension (e.g. sequence length, number of patches).\n" - "Please report any issues you encounter to help us improve it." - ) - # One encoder can serve several modalities, and must only be decorated once - for encoder_cls in dict.fromkeys(encoder_classes.values()): - self._decorate_cls_for_torch_compile( - cls=encoder_cls, - # TODO: properly infer dynamic_arg_dims based on the encoder's - # forward method signature. We assume dim 0 for all tensor inputs. - dynamic_arg_dims=None, - enable_if=should_torch_compile_mm_encoder, - is_encoder=True, - ) - - def forward( - self, - input_ids: torch.Tensor | None, - positions: torch.Tensor, - intermediate_tensors: IntermediateTensors | None = None, - inputs_embeds: torch.Tensor | None = None, - **kwargs: object, - ) -> torch.Tensor | IntermediateTensors: - # Positions shape handling for MRoPE models - if self.model_config.uses_mrope: - # [3, seq_len] -> [3, 1, seq_len] - positions = positions[:, None].contiguous() - model_output = super().forward( - input_ids, positions, intermediate_tensors, inputs_embeds - ) - return model_output - - def get_language_model(self) -> torch.nn.Module: - """Transformers modeling backend multimodal classes do not contain a separate - vLLM language model class. Therefore, in order to return a language model vLLM - class, we use a wrapper to give `self` the same interface as a text model.""" - # Exclude self and object - bases = self.__class__.mro()[1:-1] - # Keep only classes defined in `vllm.model_executor.models.transformers` - bases = [b for b in bases if ".transformers." in b.__module__] - # Exclude MultiModalMixin itself - bases = [b for b in bases if b is not MultiModalMixin] - - class LanguageModel(*bases): # type: ignore[misc] - def __init__(self, multimodal_model): - # Don't call super().__init__() to avoid re-initialization - self.__dict__.update(multimodal_model.__dict__) - - model = getattr_iter(self.model, ("language_model", "text_model"), None) - - return LanguageModel(self) - - def get_mm_mapping(self) -> MultiModelKeys: - """Get the module prefix in multimodal models""" - for name in ("language_model", "text_model"): - if getattr(self.model, name, None) is not None: - return MultiModelKeys.from_string_field(language_model=f"model.{name}") - raise ValueError( - "Could not locate the language model submodule for LoRA support" - ) - - def _split_embeddings( - self, embeddings: torch.Tensor, split_sizes: list[int] - ) -> list[torch.Tensor]: - total_expected = sum(split_sizes) - - # Flatten to 2D: [total_tokens, hidden_dim] - if embeddings.ndim > 2: - embeddings = embeddings.reshape(-1, embeddings.shape[-1]) - - total_tokens = embeddings.shape[0] - if total_tokens == total_expected: - # Direct match: split_sizes are actual token counts - token_split_sizes = split_sizes - elif total_expected > 0 and total_tokens % total_expected == 0: - # Uniform expansion: each item expands to N tokens - tokens_per_item = total_tokens // total_expected - token_split_sizes = [s * tokens_per_item for s in split_sizes] - elif total_expected > 0: - # TODO: make this an error once we know profiling never relies on it - if total_tokens == 0: - raise ValueError( - "Encoder returned empty embeddings. " - f"Expected {total_expected} tokens from " - f"split_sizes={split_sizes}" - ) - # Keep the counts out of the message: `warning_once` keys its cache on - # the args, so varying them would log on every new pair - logger.warning_once( - "Encoder returned a different number of tokens than expected; " - "padding or truncating to fit. The embeddings are not trustworthy " - "outside of memory profiling." - ) - logger.debug( - "Encoder returned %s tokens but %s were expected", - total_tokens, - total_expected, - ) - if total_tokens < total_expected: - repeat_factor = (total_expected + total_tokens - 1) // total_tokens - embeddings = embeddings.repeat(repeat_factor, 1) - embeddings = embeddings[:total_expected] - token_split_sizes = split_sizes - else: - return [] - - return list(torch.split(embeddings, token_split_sizes, dim=0)) - - def _process_audio_input(self, **kwargs) -> list[torch.Tensor] | None: - input_features: torch.Tensor | None = kwargs.pop("input_features", None) - if input_features is None: - input_features = kwargs.pop("input_values", None) - if input_features is None: - return None - - num_audio_tokens = kwargs.pop("num_audio_tokens") - kwargs.pop("token_type_ids", None) - kwargs.pop("mm_token_type_ids", None) - - # Per-audio token counts are needed as Python ints to split. - with gpu_sync_allowed(): - split_sizes = num_audio_tokens.flatten().tolist() - if isinstance(input_features, torch.Tensor): - # HuggingFace's `get_audio_features` implementations branch on - # per-sample feature lengths internally. - with gpu_sync_allowed(): - audio_output = self.model.get_audio_features( - input_features, return_dict=True, **kwargs - ) - return self._split_embeddings(audio_output.pooler_output, split_sizes) - - # Audios the processor left un-padded arrive as a list once their - # lengths differ. Encode them one at a time so that none of them is - # padded to match another. - embeddings: list[torch.Tensor] = [] - for index, features in enumerate(input_features): - audio_output = self.model.get_audio_features( - features.unsqueeze(0), - return_dict=True, - **self._select_item_kwargs(kwargs, index, len(input_features)), - ) - embeddings.extend( - self._split_embeddings(audio_output.pooler_output, [split_sizes[index]]) - ) - return embeddings - - def _process_image_input(self, **kwargs) -> list[torch.Tensor] | None: - pixel_values: torch.Tensor | None = kwargs.pop("pixel_values", None) - image_embeds: torch.Tensor | None = kwargs.pop("image_embeds", None) - # Model might use `image_patches` instead of `pixel_values` - if pixel_values is None: - pixel_values = kwargs.pop("image_patches", None) - - if image_embeds is not None: - return [image_embeds] - - if pixel_values is None: - return None - - return self._process_vision_input( - "image", pixel_values, kwargs.pop("num_image_patches"), **kwargs - ) - - def _process_video_input(self, **kwargs) -> list[torch.Tensor] | None: - pixel_values_videos: torch.Tensor | None = kwargs.pop( - "pixel_values_videos", None - ) - if pixel_values_videos is None: - return None - - kwargs.pop("second_per_grid_ts", None) - return self._process_vision_input( - "video", pixel_values_videos, kwargs.pop("num_video_patches"), **kwargs - ) - - def _process_vision_input( - self, - modality: str, - pixel_values: torch.Tensor, - num_patches: torch.Tensor, - **kwargs, - ) -> list[torch.Tensor]: - split_sizes = num_patches.flatten().tolist() - if isinstance(pixel_values, torch.Tensor): - vision_embeddings = self._get_features(modality, pixel_values, **kwargs) - if isinstance(vision_embeddings, torch.Tensor): - return self._split_embeddings(vision_embeddings, split_sizes) - return list(vision_embeddings) - - # Items the processor left un-padded arrive as a list once their - # shapes differ. Encode them one at a time so that none of them is - # padded to match another. - embeddings: list[torch.Tensor] = [] - for index, item in enumerate(pixel_values): - features = self._get_features( - modality, - item.unsqueeze(0), - **self._select_item_kwargs(kwargs, index, len(pixel_values)), - ) - # Encoders which return one entry per item return a single entry - if not isinstance(features, torch.Tensor): - features = torch.cat(list(features)) - embeddings.extend(self._split_embeddings(features, [split_sizes[index]])) - return embeddings - - def _select_item_kwargs( - self, kwargs: dict[str, Any], index: int, num_items: int - ) -> dict[str, Any]: - """Narrow the entries of `kwargs` that hold one row per item down to the item - at `index`. Length is all there is to match on, so an unrelated entry of the - same length is narrowed too.""" - return { - key: value[index : index + 1] - if isinstance(value, (torch.Tensor, list)) and len(value) == num_items - else value - for key, value in kwargs.items() - } - - def _get_features(self, modality: str, pixel_values: torch.Tensor, **kwargs) -> Any: - # grid_thw fields are registered keep_on_cpu; restore the on-device - # placement that HF get_*_features implementations expect. - for key, value in kwargs.items(): - if isinstance(value, torch.Tensor) and value.is_cpu: - kwargs[key] = async_tensor_h2d(value, pixel_values.device) - - # The underlying HuggingFace `get_*_features` implementations - # contain model-internal syncs (e.g. Idefics3 filters all-zero - # padding images via boolean-mask indexing, LlavaOnevision - # branches on per-sample batch counts). - with gpu_sync_allowed(): - get_modality_features = getattr(self.model, f"get_{modality}_features") - features = get_modality_features(pixel_values, return_dict=True, **kwargs) - return features.pooler_output - - def embed_multimodal(self, **kwargs) -> MultiModalEmbeddings: - # Each helper detects its own inputs. We are called once per modality, so the - # leftovers a helper forwards to the HF model can't belong to the other one. - embeddings: list[torch.Tensor] = [] - for process_input in ( - self._process_audio_input, - self._process_image_input, - self._process_video_input, - ): - embeddings.extend(process_input(**kwargs) or []) - return embeddings - - def get_mrope_input_positions( - self, - input_tokens: list[int], - mm_features: list[MultiModalFeatureSpec], - ) -> tuple[torch.Tensor, int]: - kwargs = MultiModalFeatureSpec.gather_kwargs( - mm_features, - { - "image_grid_thw", - "video_grid_thw", - "second_per_grid_ts", - "audio_feature_lengths", - "use_audio_in_video", - }, - ) - if kwargs.get("audio_feature_lengths") or kwargs.get("use_audio_in_video"): - raise NotImplementedError( - "Transformers modeling backend does not support audio inputs in " - "M-RoPE models." - ) - - image_grid_thw = kwargs.get("image_grid_thw", []) - video_grid_thw = kwargs.get("video_grid_thw", []) - second_per_grid_ts = kwargs.get("second_per_grid_ts", []) - - image_grid_thw = torch.stack(image_grid_thw) if image_grid_thw else None - video_grid_thw = torch.stack(video_grid_thw) if video_grid_thw else None - second_per_grid_ts = ( - torch.stack(second_per_grid_ts) if second_per_grid_ts else None - ) - - # `get_rope_index` doesn't always accept arbitrary `kwargs` - if not hasattr(self, "_get_rope_index_kwarg_names"): - import inspect - - params = inspect.signature(self.model.get_rope_index).parameters - self._get_rope_index_kwarg_names = ( - None - if any(p.kind == inspect.Parameter.VAR_KEYWORD for p in params.values()) - else frozenset(params) - ) - - kwarg_names = self._get_rope_index_kwarg_names - - def accepts_kwarg(name: str) -> bool: - return kwarg_names is None or name in kwarg_names - - # Drop a grid the model can't accept only when there is nothing to pass. - kwargs = { - name: value - for name, value in ( - ("image_grid_thw", image_grid_thw), - ("video_grid_thw", video_grid_thw), - ("second_per_grid_ts", second_per_grid_ts), - ) - if value is not None or accepts_kwarg(name) - } - if accepts_kwarg("mm_token_type_ids"): - mm_token_type_ids = torch.zeros(len(input_tokens), dtype=torch.int) - for feature in mm_features: - position = feature.mm_position - offset, length = position.offset, position.length - is_embed = position.is_embed - if is_embed is None: - is_embed = slice(None) - mm_token_type_id = _MODALITY_TO_TOKEN_TYPE_ID[feature.modality] - mm_token_type_ids[offset : offset + length][is_embed] = mm_token_type_id - kwargs["mm_token_type_ids"] = mm_token_type_ids.unsqueeze(0) - - mrope_positions, mrope_position_delta = self.model.get_rope_index( - input_ids=torch.tensor(input_tokens).unsqueeze(0), - **kwargs, - ) - - mrope_positions = mrope_positions[:, 0] - mrope_position_delta = mrope_position_delta[0].item() - - return mrope_positions, mrope_position_delta diff --git a/vllm/model_executor/models/transformers/utils.py b/vllm/model_executor/models/transformers/utils.py deleted file mode 100644 index d36ff2015b..0000000000 --- a/vllm/model_executor/models/transformers/utils.py +++ /dev/null @@ -1,407 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -# Copyright 2024 The vLLM team. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Transformers modeling backend utilities.""" - -from collections.abc import Callable, Iterator -from contextlib import contextmanager -from itertools import chain -from operator import attrgetter -from pathlib import Path -from typing import TYPE_CHECKING, Literal, TypeAlias, TypeVar - -import torch -from torch import nn - -from vllm.logger import init_logger -from vllm.model_executor.layers.conv import Conv2dLayer, Conv3dLayer -from vllm.model_executor.layers.layernorm import LayerNorm -from vllm.model_executor.layers.linear import ( - ColumnParallelLinear, - ReplicatedLinear, - RowParallelLinear, -) -from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding -from vllm.model_executor.models.utils import maybe_prefix -from vllm.transformers_utils.config import iter_rope_parameters - -if TYPE_CHECKING: - from vllm.config import VllmConfig - from vllm.model_executor.layers.quantization import QuantizationConfig - - -logger = init_logger(__name__) - - -# Copied from `accelerate` -@contextmanager -def init_on_device_without_buffers(device: torch.device): - """A context manager under which models are initialized with all - parameters on the specified device. However buffers are not - initialized on specified device. - - Args: - device (`torch.device`): - Device to initialize all parameters on. - - """ - old_register_parameter = nn.Module.register_parameter - - def register_empty_parameter(module, name, param): - old_register_parameter(module, name, param) - if param is not None: - param_cls = type(module._parameters[name]) - kwargs = module._parameters[name].__dict__ - kwargs["requires_grad"] = param.requires_grad - module._parameters[name] = param_cls( - module._parameters[name].to(device), **kwargs - ) - - tensor_constructors_to_patch: dict[str, Callable] = {} - - def patch_tensor_constructor(fn): - def wrapper(*args, **kwargs): - kwargs["device"] = device - return fn(*args, **kwargs) - - return wrapper - - try: - nn.Module.register_parameter = register_empty_parameter - for torch_function_name in tensor_constructors_to_patch: - setattr( - torch, - torch_function_name, - patch_tensor_constructor(getattr(torch, torch_function_name)), - ) - yield - finally: - nn.Module.register_parameter = old_register_parameter - for ( - torch_function_name, - old_torch_function, - ) in tensor_constructors_to_patch.items(): - setattr(torch, torch_function_name, old_torch_function) - - -Style = Literal[ - "colwise", - "rowwise", - "replicate", - "colwise_gather_output", - "rowwise_split_input", -] - - -def replace_linear_class( - linear: nn.Linear, - style: Style = "replicate", - quant_config: "QuantizationConfig | None" = None, - *, - prefix: str = "", -) -> ColumnParallelLinear | RowParallelLinear | ReplicatedLinear: - """Replace nn.Linear with one of vLLM's tensor parallel linear classes. - - Args: - linear: `nn.Linear` to be replaced. - style: Tensor parallel style of the new linear, e.g. "colwise". - quant_config: Quantization config for the new linear. - prefix: Module prefix of the new linear, used for quantization lookup. - - Returns: - The new linear. - - """ - if not isinstance(style, str): - raise ValueError(f"Unsupported parallel style type {type(style)}, expected str") - - vllm_linear_cls, vllm_linear_kwargs = { - "colwise": (ColumnParallelLinear, {}), - "rowwise": (RowParallelLinear, {}), - "replicate": (ReplicatedLinear, {}), - "colwise_gather_output": (ColumnParallelLinear, {"gather_output": True}), - "rowwise_split_input": (RowParallelLinear, {"input_is_parallel": False}), - }.get(style, (ReplicatedLinear, {})) - - return vllm_linear_cls( - input_size=linear.in_features, - output_size=linear.out_features, - bias=linear.bias is not None, - quant_config=quant_config, - prefix=prefix, - return_bias=False, - **vllm_linear_kwargs, - ) - - -TorchConv: TypeAlias = nn.Conv2d | nn.Conv3d -VllmConv: TypeAlias = Conv2dLayer | Conv3dLayer - - -def replace_conv_class(conv: TorchConv) -> VllmConv | TorchConv: - """Replace a Transformers Conv2d/Conv3d with vLLM's Conv2d/Conv3d. - - Args: - conv: `nn.Conv2d` or `nn.Conv3d` to be replaced. - - Returns: - The new `Conv2dLayer` or `Conv3dLayer`. If the conv module is not supported, - returns the original conv module. - - """ - # vLLM does not handle non-zero padding modes - if conv.padding_mode != "zeros": - return conv - - vllm_conv_cls = { - nn.Conv2d: Conv2dLayer, - nn.Conv3d: Conv3dLayer, - }.get(type(conv)) - - if vllm_conv_cls is None: - return conv - - return vllm_conv_cls( - in_channels=conv.in_channels, - out_channels=conv.out_channels, - kernel_size=conv.kernel_size, - stride=conv.stride, - padding=conv.padding, - dilation=conv.dilation, - groups=conv.groups, - bias=conv.bias is not None, - padding_mode=conv.padding_mode, - params_dtype=conv.weight.dtype, - ) - - -def replace_layernorm_class(layernorm: nn.LayerNorm) -> nn.Module: - """Replace a standard (mean-centered) `nn.LayerNorm` with vLLM's - `LayerNorm`. - - Args: - layernorm: `nn.LayerNorm` to be replaced. - - Returns: - The new `LayerNorm`. If the layernorm is not supported (a subclass with - its own behavior, multi-dim `normalized_shape`, or not both - elementwise-affine and biased), returns the original module unchanged. - - """ - if ( - type(layernorm) is not nn.LayerNorm - or len(layernorm.normalized_shape) != 1 - or not layernorm.elementwise_affine - or layernorm.bias is None - ): - return layernorm - - return LayerNorm( - layernorm.normalized_shape[0], - eps=layernorm.eps, - dtype=layernorm.weight.dtype, - ) - - -def attrsetter(attr: str) -> Callable[[object, object], None]: - """Set a possibly nested attribute, like the inverse of attrgetter.""" - parent, _, name = attr.rpartition(".") - - def setter(obj: object, value: object): - attr_parent = attrgetter(parent)(obj) if parent else obj - setattr(attr_parent, name, value) - - return setter - - -class _UninitializedEmbedding(nn.Embedding): - """Make `__init__` inert, so that `VocabParallelEmbedding.__init__`' - call to `super().__init__` does not invoke `nn.Embedding.__init__`.""" - - def __init__(self): - pass - - -class _VocabParallelEmbeddingBase(VocabParallelEmbedding, _UninitializedEmbedding): - """Orders `VocabParallelEmbedding` ahead of `nn.Embedding` in the MRO, so that - `super().forward(...)` in an `nn.Embedding` subclass reaches vLLM's embedding.""" - - -_rebased_embedding_classes: dict[type[nn.Embedding], type[VocabParallelEmbedding]] = {} - - -def _rebase_on_vocab_parallel(cls: type[nn.Embedding]) -> type[VocabParallelEmbedding]: - """Subclass `cls` so that `VocabParallelEmbedding` supersedes its `nn.Embedding`. - - Args: - cls: The `nn.Embedding` subclass to rebase. Cached, so a given `cls` always - maps to the same class. - - Returns: - The new class, to assign to `__class__` of an instance of `cls`. - - """ - if cls not in _rebased_embedding_classes: - rebased_cls = type(cls.__name__, (cls, _VocabParallelEmbeddingBase), {}) - _rebased_embedding_classes[cls] = rebased_cls - return _rebased_embedding_classes[cls] - - -def replace_embedding_class( - embedding: nn.Embedding, - quant_config: "QuantizationConfig | None" = None, - *, - prefix: str = "", -) -> nn.Module: - """Replace `nn.Embedding` with `VocabParallelEmbedding`. - - Args: - embedding: The `nn.Embedding` holding a vocab table. - quant_config: Quantization config for the new embedding. - prefix: Qualname of `embedding`, used to look up its quantization method. - - Returns: - The module to install in `embedding`'s place. - - """ - # Tied tables are reached more than once, so don't rebase an already new embedding - if isinstance(embedding, VocabParallelEmbedding): - return embedding - - kwargs = dict( - num_embeddings=embedding.num_embeddings, - embedding_dim=embedding.embedding_dim, - params_dtype=embedding.weight.dtype, - quant_config=quant_config, - prefix=prefix, - ) - # If `embedding` is a bare `nn.Embedding`, simple replace - if type(embedding) is nn.Embedding: - return VocabParallelEmbedding(**kwargs) - - # Otherwise `embedding` inherits `nn.Embedding`, rebase it in place - embedding.__class__ = _rebase_on_vocab_parallel(type(embedding)) - VocabParallelEmbedding.__init__(embedding, **kwargs) - return embedding - - -def recursive_replace_linear( - model: nn.Module, - quant_config: "QuantizationConfig | None", - prefix: str = "", -): - """Recursively replace linear modules in the model as needed.""" - - def _recursive_replace(module: nn.Module, prefix: str): - for child_name, child_module in module.named_children(): - new_module = child_module - qual_name = maybe_prefix(prefix, child_name) - # Replace modules as needed - if isinstance(child_module, nn.Linear): - new_module = replace_linear_class( - child_module, - "replicate", - quant_config, - prefix=qual_name, - ) - else: - _recursive_replace(child_module, prefix=qual_name) - if new_module is not child_module: - setattr(module, child_name, new_module) - - _recursive_replace(model, prefix=prefix) - - -T = TypeVar("T") - - -def maybe_per_layer(value: T | list[T], layer_idx: int) -> T: - """Pick `layer_idx`'s entry from a config field that may be sized per layer.""" - return value[layer_idx] if isinstance(value, list) else value - - -def named_state(module: nn.Module) -> Iterator[tuple[str, torch.Tensor]]: - """`module`'s own state (i.e. named parameters and buffers).""" - return chain(module.named_parameters(), module.named_buffers()) - - -def log_replacement(name: str, old_module: nn.Module, new_module: nn.Module): - logger.debug("%s: %s -> %s", name, old_module, new_module) - - -def get_feature_request_tip( - model: str, - trust_remote_code: bool, -) -> str: - hf_url = f"a discussion at https://huggingface.co/{model}/discussions/new" - gh_url = "an issue at https://github.com/huggingface/transformers/issues/new/choose" - url = hf_url if trust_remote_code else gh_url - prefix = f"Please open {url} to request support for this feature. " - if Path(model).exists(): - prefix = "" - doc_url = "https://docs.vllm.ai/en/latest/models/supported_models.html#writing-custom-models" - tip = f"See {doc_url} for instructions on how to add support yourself." - return f"{prefix}{tip}" - - -def dynamic_rope_update_is_noop(vllm_config: "VllmConfig") -> bool: - """Whether `max_model_len` keeps Transformers' dynamic rope from ever rescaling. - - `@dynamic_rope_update` only recomputes `inv_freq` once positions grow past - `max_position_embeddings`, which vLLM never feeds beyond `max_model_len`. - """ - text_config = vllm_config.model_config.hf_config.get_text_config() - max_position_embeddings = getattr(text_config, "max_position_embeddings", None) - if max_position_embeddings is None: - return False - return vllm_config.model_config.max_model_len <= max_position_embeddings - - -def can_enable_torch_compile(vllm_config: "VllmConfig") -> bool: - """Callable to be passed to `@support_torch_compile`'s `enable_if` argument. - - Defaults to `True` but is disabled in the following situations: - - - The model uses dynamic rope scaling that can still rescale within - `max_model_len` (otherwise `disable_noop_dynamic_rope` removes it). - """ - text_config = vllm_config.model_config.hf_config.get_text_config() - rope_types = {rp["rope_type"] for rp in iter_rope_parameters(text_config)} - if "dynamic" not in rope_types: - return True - # Layer types using longrope still sync, even once dynamic rope is disabled - return "longrope" not in rope_types and dynamic_rope_update_is_noop(vllm_config) - - -def disable_noop_dynamic_rope(model: nn.Module, vllm_config: "VllmConfig") -> None: - """Skip `@dynamic_rope_update` in rotary modules where it can never rescale. - - The update syncs on `torch.max(position_ids)` every forward, which breaks - torch.compile and CUDA graph capture. Relabelling the module's `rope_type` as - "default" skips it while keeping the `inv_freq` computed at init. - """ - if not dynamic_rope_update_is_noop(vllm_config): - return - for module in model.modules(): - rope_type = getattr(module, "rope_type", None) - if not hasattr(module, "max_seq_len_cached") or rope_type is None: - continue - if isinstance(rope_type, dict): - module.rope_type = { - k: "default" if v == "dynamic" else v for k, v in rope_type.items() - } - elif rope_type == "dynamic": - module.rope_type = "default" diff --git a/vllm/model_executor/warmup/kernel_warmup.py b/vllm/model_executor/warmup/kernel_warmup.py index 0a9893496e..2c972d7458 100644 --- a/vllm/model_executor/warmup/kernel_warmup.py +++ b/vllm/model_executor/warmup/kernel_warmup.py @@ -6,7 +6,6 @@ happen during model execution. """ import json -import sys import time from pathlib import Path from typing import TYPE_CHECKING @@ -134,25 +133,6 @@ def _warmup_bf16x3_router_gemm( logger.info_once("Warmed up BF16x3 router GEMM configs: %s.", configs) -def _warmup_gemm_rs_ar() -> None: - # Model construction (Kimi-K3, DeepSeek-V4.1) imports this module only - # when GEMM-RS/AR is enabled and initializes its singleton before - # kernel_warmup runs. Avoid importing it here so other models do not - # compile the RS/AR variants. - module = sys.modules.get("vllm.model_executor.kernels.linear.cute_dsl.gemm_rs_ar") - if module is None: - return - compiled = module.warmup_gemm_rs_ar() - if compiled: - logger.info_once("Warmed up %d GEMM-RS/AR variants.", compiled) - - -def _autotune_kimi_k3_kda_qkvg(model: torch.nn.Module) -> None: - module = sys.modules.get("vllm.models.kimi_k3.nvidia.low_latency_gemm") - if module is not None: - module.autotune_kda_qkvg(model) - - def kernel_warmup(worker: "Worker", *, process_local_only: bool = False): if not worker.use_v2_model_runner: # The KV-block zeroing kernel is driven by the scheduler's @@ -202,8 +182,6 @@ def kernel_warmup(worker: "Worker", *, process_local_only: bool = False): if current_platform.has_device_capability(90): _warmup_ll_bf16_router_gemm(worker.get_model()) - _warmup_gemm_rs_ar() - if worker.vllm_config.kernel_config.enable_cutedsl_warmup: # TODO(roberto): Remove after registered CuTeDSL warmups are migrated # to the shared JIT warmup infrastructure. @@ -478,16 +456,11 @@ def flashinfer_autotune(runner: "GPUModelRunner", *, skip_attn: bool = False) -> ) skip_attn = skip_attn or hisparse_enabled if hisparse_enabled: - # HiSparse hot-buffer attention is bounded by decode batch - # size, not the prefill-sized batch used for the full model. - from vllm.model_executor.warmup.flashinfer_sparse_mla_warmup import ( - autotune_hisparse_flashinfer_attention, - ) - - autotune_hisparse_flashinfer_attention(runner) + # HiSparse config still bounds decode-batch autotune even + # though the connector was removed from this build. + logger.info_once("HiSparse configured; skipping attention autotune.") _run_flashinfer_autotune_dummy_runs(runner, skip_attn=skip_attn) replayssm_autotune_warmup(runner) - _autotune_kimi_k3_kda_qkvg(runner.get_model()) finally: set_autotune_process_group(None) diff --git a/vllm/models/qwen4_exp/amd/__init__.py b/vllm/models/qwen4_exp/amd/__init__.py deleted file mode 100644 index 208f01a7cb..0000000000 --- a/vllm/models/qwen4_exp/amd/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project diff --git a/vllm/models/qwen4_exp/amd/hyperconnection.py b/vllm/models/qwen4_exp/amd/hyperconnection.py deleted file mode 100644 index 087f03fdcf..0000000000 --- a/vllm/models/qwen4_exp/amd/hyperconnection.py +++ /dev/null @@ -1,205 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""HyperConnection (Gated Residual) utilities for the AMD model variant. - -Implements the HyperConnection residual scheme proposed in -"HyperConnections" (https://arxiv.org/abs/2409.19606). This AMD variant -delays each HC combine to the following HC mix boundary. HC glue kernels, -including fused combine+RMSNorm, live in ``ops/hc.py``; projections remain -standard vLLM Linear modules. - -Hidden states between layers have shape ``[..., HC*HS]`` with HS inner -(HC outer, HS inner — checkpoint-native layout). - -Typical usage inside a transformer decoder layer:: - - self.attn_hc = GatedResidual(hc_config) - - hidden_states, block_input, injection = self.attn_hc.mix(hidden_states) - attention_output = attention(block_input) - hidden_states, block_input, injection = self.mlp_hc.combine_and_mix( - hidden_states, attention_output, injection - ) -""" - -import torch -from torch import nn - -from vllm.model_executor.layers.linear import ( - MergedColumnParallelLinear, - ReplicatedLinear, -) -from vllm.model_executor.models.utils import maybe_prefix - -from ..common.hyperconnection import ( - GroupedGemmaRMSNorm, - HyperConnectionConfig, -) -from .ops.hc import ( - grouped_gemma_rmsnorm, - hc_combine, - hc_combine_norm, - hc_gate_mix, - hc_silu, -) - - -# --------------------------------------------------------------------------- -# Gated-residual variant -# --------------------------------------------------------------------------- -class GatedResidual(nn.Module): - """Gated HyperConnection with learnable low-rank mixing and injection. - - ``combine_and_mix()`` runs the pre pipeline (grouped GemmaRMSNorm -> merged - low-rank down+inject GEMM -> silu -> up GEMM -> sigmoid -> gated mean - over the HC streams). When passed a pending block output and an injection, - it fuses their residual combine with the RMSNorm. Final mixers use - ``use_combine=False`` and do not produce a new injection. - - Weights: the norm owns the grouped GemmaRMSNorm affine; the projections - are vLLM Linear modules (merged replicated linear for down+inject), so - GEMM dispatch (e.g. the low-latency skinny GEMM) applies through the - standard quant_method mechanism. - """ - - def __init__( - self, - config: HyperConnectionConfig, - use_combine: bool = True, - prefix: str = "", - ) -> None: - super().__init__() - self.config = config - self.lora_rank = config.hc_lowrank - self.hc_count = config.hc_count - self.hidden_size = config.hidden_size - self.use_combine = use_combine - - norm_size = ( - self.hyper_hidden_size if config.hc_per_branch_norm else config.hidden_size - ) - group_size = config.hidden_size if config.hc_per_branch_norm else None - # Normalize each H-sized HC stream independently while retaining a - # separate affine weight for every element of the HC*H layout. - self.hc_norm = GroupedGemmaRMSNorm( - norm_size, - eps=config.rms_norm_eps, - group_size=group_size, - dtype=config.params_dtype, - ) - - # -- vLLM Linear weights -------------------------------------------- - # The merged skinny-GEMM shape is physically padded to 16 rows for - # alignment and efficient backend dispatch. - self.pad_size = (-(self.lora_rank + self.hc_count)) % 16 if use_combine else 0 - if use_combine: - self.input_mix_weight_down_block_inject = MergedColumnParallelLinear( - self.hyper_hidden_size, - [self.lora_rank, self.hc_count] - + ([self.pad_size] if self.pad_size else []), - bias=False, - params_dtype=config.params_dtype, - quant_config=None, - prefix=maybe_prefix(prefix, "input_mix_weight_down_block_inject"), - return_bias=False, - disable_tp=True, - ) - else: - self.input_mix_weight_down = ReplicatedLinear( - self.hyper_hidden_size, - self.lora_rank, - bias=False, - params_dtype=config.params_dtype, - quant_config=None, - prefix=maybe_prefix(prefix, "input_mix_weight_down"), - return_bias=False, - ) - self.input_mix_weight_up = ReplicatedLinear( - self.lora_rank, - self.hyper_hidden_size, - bias=False, - params_dtype=config.params_dtype, - quant_config=None, - prefix=maybe_prefix(prefix, "input_mix_weight_up"), - return_bias=False, - ) - - def mix( - self, hidden_states: torch.Tensor - ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None]: - xn = grouped_gemma_rmsnorm( - hidden_states, - self.hc_norm.weight, - self.config.rms_norm_eps, - self.hc_count, - ) - - if self.use_combine: - # produce injection logits for combine - split_sizes = [self.lora_rank, self.hc_count, self.pad_size] - down_and_injection = self.input_mix_weight_down_block_inject(xn) - lora, injection, _ = down_and_injection.split(split_sizes, dim=-1) - else: - lora = self.input_mix_weight_down(xn) - injection = None - - lora = hc_silu(lora, self.hc_count) - gate = self.input_mix_weight_up(lora) # [M, D] - block_input = hc_gate_mix(xn, gate, self.hc_count) - - return hidden_states, block_input, injection - - def combine_and_mix( - self, - hidden_states: torch.Tensor, - prev_block_output: torch.Tensor, - prev_injection: torch.Tensor, - ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None]: - """Consume a pending combine, then prepare the next block input. - - ``hidden_states`` is the multi-stream state from before the pending - block's mix. Its combine with ``block_output`` is fused with this - module's input RMSNorm. - """ - hidden_states, xn = hc_combine_norm( - hidden_states, - prev_block_output, - prev_injection, - self.hc_norm.weight, - self.config.rms_norm_eps, - self.hc_count, - ) - - if self.use_combine: - # produce injection logits for combine - split_sizes = [self.lora_rank, self.hc_count, self.pad_size] - down_and_injection = self.input_mix_weight_down_block_inject(xn) - lora, injection, _ = down_and_injection.split(split_sizes, dim=-1) - else: - lora = self.input_mix_weight_down(xn) - injection = None - - lora = hc_silu(lora, self.hc_count) - gate = self.input_mix_weight_up(lora) # [M, D] - block_input = hc_gate_mix(xn, gate, self.hc_count) - - return hidden_states, block_input, injection - - def combine( - self, - hidden_states: torch.Tensor, - block_output: torch.Tensor, - injection: torch.Tensor, - ) -> torch.Tensor: - return hc_combine(hidden_states, block_output, injection, self.hc_count) - - @property - def hyper_hidden_size(self) -> int: - return self.hc_count * self.hidden_size - - -__all__ = [ - "GatedResidual", - "GroupedGemmaRMSNorm", - "HyperConnectionConfig", -] diff --git a/vllm/models/qwen4_exp/amd/indexer_qsa.py b/vllm/models/qwen4_exp/amd/indexer_qsa.py deleted file mode 100644 index 09cb39acf9..0000000000 --- a/vllm/models/qwen4_exp/amd/indexer_qsa.py +++ /dev/null @@ -1,420 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Qwen4Exp weight-free QSA indexer.""" - -from __future__ import annotations - -from typing import TYPE_CHECKING, cast - -import torch -from torch import nn -from transformers import Qwen4ExpTextConfig - -from vllm.config import VllmConfig -from vllm.forward_context import get_forward_context -from vllm.model_executor.layers.layernorm import GemmaRMSNorm -from vllm.model_executor.layers.linear import ReplicatedLinear -from vllm.model_executor.layers.quantization import QuantizationConfig -from vllm.model_executor.layers.rotary_embedding.mrope import triton_mrope - -from ..common.qsa_cache import ( - QSACompressedKeyCache, - QSAForwardMetadata, - QSAKeyStateCache, - canonical_qsa_rope_positions, -) -from .ops.qsa_prepare import qsa_prepare, supports_fused_pre_indexer - -if TYPE_CHECKING: - from .qsa import Qwen4ExpQSAAttention - - -def apply_qsa_rope( - rotary_emb: nn.Module, - positions: torch.Tensor, - tensor: torch.Tensor, -) -> torch.Tensor: - """Apply the main attention's exact 1D/MRoPE composition to QSA heads.""" - num_tokens, _, head_dim = tensor.shape - rotary_dim = rotary_emb.rotary_dim - cache = rotary_emb._match_cos_sin_cache_dtype(tensor) # noqa: SLF001 - cos_sin = cache[positions] - cos, sin = cos_sin.chunk(2, dim=-1) - if positions.ndim == 2: - shape = tensor.shape - tensor, _ = triton_mrope( - tensor.reshape(num_tokens, -1), - tensor.new_empty((num_tokens, head_dim)), - cos, - sin, - rotary_emb.mrope_section, - head_dim, - rotary_dim, - rotary_emb.mrope_interleaved, - rotary_emb.is_neox_style, - ) - return tensor.reshape(shape) - - rotated = rotary_emb.apply_rotary_emb( - tensor[..., :rotary_dim], - cos, - sin, - ) - return torch.cat((rotated, tensor[..., rotary_dim:]), dim=-1) - - -def apply_qsa_rmsnorm( - norm: GemmaRMSNorm, - tensor: torch.Tensor, -) -> torch.Tensor: - """Use vLLM's portable RMSNorm implementation on ROCm.""" - return cast(torch.Tensor, norm(tensor)) - - -class QSAIndexer(nn.Module): - """Replicated Q/K projection plus paged, weight-free QSA selection. - - ``prefix`` must be the checkpoint's indexer prefix, normally - ``model.layers.N.self_attn.indexer``. Consequently the trainable names are - ``index_qk_proj``, ``q_layernorm`` and ``k_layernorm`` under that prefix. - """ - - def __init__( - self, - *, - vllm_config: VllmConfig, - config: Qwen4ExpTextConfig, - layer_id: int, - rotary_emb: nn.Module, - quant_config: QuantizationConfig | None = None, - prefix: str = "", - ) -> None: - super().__init__() - if vllm_config.cache_config is None: - raise ValueError("QSA requires a paged KV cache") - if vllm_config.model_config.dtype != torch.bfloat16: - raise NotImplementedError("Qwen4Exp QSA currently requires BF16") - - self.layer_id = int(layer_id) - self.index_n_heads = int(config.indexer_n_heads) - self.index_kv_heads = int(config.indexer_kv_heads) - self.index_head_dim = int(config.indexer_head_dim) - self.token_topk = int(config.indexer_budget) - self.compress_ratio = int(config.indexer_compress_ratio) - self.rotary_emb = rotary_emb - self.use_fused_pre_indexer = supports_fused_pre_indexer( - rotary_emb, - self.index_head_dim, - self.index_kv_heads, - self.compress_ratio, - ) - self.prefix = prefix - # MTP step 0 selects the target-aligned rows; later steps reuse them - # while continuing to update the QSA side cache. - self.skip_topk = False - - self.index_qk_proj = ReplicatedLinear( - int(config.hidden_size), - (self.index_n_heads + self.index_kv_heads) * self.index_head_dim, - bias=False, - quant_config=quant_config, - prefix=f"{prefix}.index_qk_proj" if prefix else "index_qk_proj", - ) - self.q_layernorm = GemmaRMSNorm( - self.index_head_dim, - eps=float(getattr(config, "rms_norm_eps", 1e-6)), - ) - self.k_layernorm = GemmaRMSNorm( - self.index_head_dim, - eps=float(getattr(config, "rms_norm_eps", 1e-6)), - ) - - cache_config = vllm_config.cache_config - cache_prefix = f"{prefix}." if prefix else "" - self.raw_key_cache = QSAKeyStateCache( - head_size=self.index_head_dim, - dtype=torch.bfloat16, - cache_rope_positions=vllm_config.model_config.uses_mrope, - prefix=f"{cache_prefix}raw_key_cache", - cache_config=cache_config, - compress_ratio=self.compress_ratio, - vllm_config=vllm_config, - ) - self.compressed_key_cache = QSACompressedKeyCache( - head_size=self.index_head_dim, - dtype=torch.bfloat16, - compress_ratio=self.compress_ratio, - prefix=f"{cache_prefix}compressed_key_cache", - cache_config=cache_config, - vllm_config=vllm_config, - ) - - @property - def output_width(self) -> int: - return self.token_topk + self.compress_ratio - 1 - - def project( - self, - hidden_states: torch.Tensor, - ) -> tuple[torch.Tensor, torch.Tensor]: - """Project replicated Q/K and split, leaving both operands unrotated.""" - qk, _ = self.index_qk_proj(hidden_states) - return qk.split( - ( - self.index_n_heads * self.index_head_dim, - self.index_kv_heads * self.index_head_dim, - ), - dim=-1, - ) - - def project_qk( - self, - hidden_states: torch.Tensor, - positions: torch.Tensor, - ) -> tuple[torch.Tensor, torch.Tensor]: - """Project replicated Q/K, normalize+rotate Q, and preserve raw K.""" - q_raw, token_k = self.project(hidden_states) - q = q_raw.reshape(-1, self.index_n_heads, self.index_head_dim) - q = apply_qsa_rmsnorm( - self.q_layernorm, - q.reshape(-1, self.index_head_dim), - ).reshape_as(q) - q = apply_qsa_rope(self.rotary_emb, positions, q) - return q, token_k.reshape(-1, 1, self.index_head_dim) - - def normalize_compressed_keys( - self, - compressed_keys: torch.Tensor, - first_rope_positions: torch.Tensor, - ) -> torch.Tensor: - """Normalize pooled K and apply the first token's exact group position.""" - keys = compressed_keys.reshape(-1, self.index_head_dim) - keys = apply_qsa_rmsnorm(self.k_layernorm, keys).reshape( - -1, 1, self.index_head_dim - ) - if getattr(self.rotary_emb, "mrope_section", None): - positions = first_rope_positions.transpose(0, 1) - else: - positions = first_rope_positions[:, 0] - return apply_qsa_rope(self.rotary_emb, positions, keys) - - def _metadata( - self, - ) -> tuple[QSAForwardMetadata, QSAForwardMetadata] | None: - metadata = get_forward_context().attn_metadata - if isinstance(metadata, list): - metadata = metadata[0] - if not isinstance(metadata, dict): - return None - raw = cast(QSAForwardMetadata, metadata[self.raw_key_cache.prefix]) - compressed = cast( - QSAForwardMetadata, metadata[self.compressed_key_cache.prefix] - ) - if raw.num_actual_tokens != compressed.num_actual_tokens: - raise RuntimeError("QSA side-cache metadata token counts disagree") - if not raw.logical_positions.is_cuda and ( - not torch.equal(raw.logical_positions, compressed.logical_positions) - ): - raise RuntimeError("QSA side-cache metadata positions disagree") - return raw, compressed - - def _update_and_compress( - self, - token_k: torch.Tensor, - positions: torch.Tensor, - raw_metadata: QSAForwardMetadata, - compressed_metadata: QSAForwardMetadata, - ) -> None: - num_tokens = raw_metadata.num_actual_tokens - raw_key_cache = self.raw_key_cache.key_cache - rope_position_cache = self.raw_key_cache.rope_position_cache - from .ops.qsa import qsa_compress_groups_with_ratio, qsa_store_cache_rows - - if rope_position_cache is None: - position_rows = raw_metadata.logical_positions.view(-1, 1, 1).expand( - -1, 1, 3 - ) - else: - position_rows = canonical_qsa_rope_positions(positions)[:num_tokens].to( - device=raw_key_cache.device - ) - pooled, first_positions = qsa_compress_groups_with_ratio( - token_k[:num_tokens], - position_rows, - raw_key_cache, - raw_metadata.block_table, - raw_metadata.token_to_req, - raw_metadata.query_start_loc, - raw_metadata.logical_positions, - compressed_metadata.slot_mapping, - self.compress_ratio, - rope_position_cache, - ) - normalized = self.normalize_compressed_keys(pooled, first_positions) - qsa_store_cache_rows( - self.compressed_key_cache.kv_cache, - compressed_metadata.slot_mapping, - normalized, - ) - qsa_store_cache_rows( - raw_key_cache, - raw_metadata.slot_mapping, - token_k[:num_tokens], - ) - if rope_position_cache is not None: - qsa_store_cache_rows( - rope_position_cache, - raw_metadata.slot_mapping, - position_rows, - ) - - def _fused_prepare( - self, - projected_q: torch.Tensor, - raw_keys: torch.Tensor, - positions: torch.Tensor, - raw_metadata: QSAForwardMetadata, - compressed_metadata: QSAForwardMetadata, - attn: Qwen4ExpQSAAttention, - qkv: torch.Tensor, - slot_mapping: torch.Tensor, - gate_out: torch.Tensor | None, - ) -> tuple[torch.Tensor, tuple[torch.Tensor, torch.Tensor]]: - """Prepare the indexer and ``attn``'s main attention in one launch. - - Normalizes and rotates the index Q, compresses K and writes both side - caches, which replaces ``project_qk``'s norm/RoPE half and the whole of - ``_update_and_compress``. The same launch applies ``attn``'s QK-norm and - RoPE, copies its gate and writes its K/V into ``attn.kv_cache``, - replacing ``_project_qkv_gate`` and ``do_kv_cache_update``. Both halves - must stay numerically interchangeable with what they replace. - """ - num_tokens = raw_metadata.num_actual_tokens - raw_key_cache = self.raw_key_cache - q = projected_q.new_empty(num_tokens, self.index_n_heads, self.index_head_dim) - main_outputs = qsa_prepare( - projected_q, - raw_keys, - positions, - self.rotary_emb.cos_sin_cache, - self.q_layernorm.weight, - self.k_layernorm.weight, - self.q_layernorm.variance_epsilon, - q, - raw_key_cache.kv_cache, - raw_metadata.slot_mapping, - raw_metadata.block_table, - raw_metadata.query_start_loc, - raw_metadata.logical_positions, - self.compressed_key_cache.kv_cache, - compressed_metadata.slot_mapping, - compressed_metadata.k_work_metadata, - compress_ratio=self.compress_ratio, - mrope_section=getattr(self.rotary_emb, "mrope_section", None), - rope_pos_offset=( - raw_key_cache.rope_position_offset - if raw_key_cache.cache_rope_positions - else None - ), - main_qkv=qkv[:num_tokens], - main_q_norm_weight=attn.q_norm.weight, - main_k_norm_weight=attn.k_norm.weight, - main_eps=attn.q_norm.variance_epsilon, - main_kv_cache=attn.kv_cache.transpose(1, 2), - main_slot_mapping=slot_mapping[:num_tokens], - main_k_scale=attn._k_scale_float, - main_v_scale=attn._v_scale_float, - main_gate_out=None if gate_out is None else gate_out[:num_tokens], - ) - return q, main_outputs - - def _select( - self, - q: torch.Tensor, - metadata: QSAForwardMetadata, - out: torch.Tensor | None, - ) -> torch.Tensor: - from .ops.qsa import qsa_select_paged_tokens - - return qsa_select_paged_tokens( - q, - self.compressed_key_cache.kv_cache, - metadata.block_table, - metadata.token_to_req, - metadata.logical_positions, - metadata.seq_lens, - self.token_topk, - self.compress_ratio, - out, - ) - - def forward( - self, - hidden_states: torch.Tensor, - positions: torch.Tensor, - out: torch.Tensor | None = None, - *, - attn: Qwen4ExpQSAAttention, - qkv: torch.Tensor | None = None, - slot_mapping: torch.Tensor | None = None, - gate_out: torch.Tensor | None = None, - ) -> tuple[torch.Tensor, tuple[torch.Tensor, torch.Tensor] | None]: - """Return fixed-width request-relative token indices padded with ``-1``. - - With ``attn.use_fused_qsa_prepare``, the same launch also writes - ``attn``'s K/V into ``attn.kv_cache`` at ``slot_mapping`` and prepares - its Q and gate from ``qkv``, returned as the second element (None - otherwise). The gate is written into ``gate_out`` when given. ``qkv``, - ``slot_mapping`` and ``gate_out`` are only read in that mode. - """ - metadata = self._metadata() - if metadata is None: - # Preserve step-0 indices when later MTP steps reuse the buffer. - if self.skip_topk and out is not None: - return out, None - result = torch.full( - (hidden_states.shape[0], self.output_width), - -1, - dtype=torch.int32, - device=hidden_states.device, - ) - if out is not None: - out.copy_(result) - return out, None - return result, None - raw_metadata, compressed_metadata = metadata - num_tokens = raw_metadata.num_actual_tokens - hidden_states = hidden_states[:num_tokens] - positions = positions[..., :num_tokens] - main_outputs: tuple[torch.Tensor, torch.Tensor] | None = None - if attn.use_fused_qsa_prepare: - if qkv is None or slot_mapping is None: - raise ValueError("fused QSA prepare requires qkv and slot_mapping") - projected_q, raw_keys = self.project(hidden_states) - q, main_outputs = self._fused_prepare( - projected_q, - raw_keys, - positions, - raw_metadata, - compressed_metadata, - attn, - qkv, - slot_mapping, - gate_out, - ) - else: - q, token_k = self.project_qk(hidden_states, positions) - self._update_and_compress( - token_k, - positions, - raw_metadata, - compressed_metadata, - ) - if self.skip_topk: - if out is None: - raise RuntimeError("QSA top-k reuse requires an output buffer") - return out, main_outputs - return self._select(q, compressed_metadata, out), main_outputs - - -__all__ = ["QSAIndexer", "apply_qsa_rope"] diff --git a/vllm/models/qwen4_exp/amd/low_latency_gemm.py b/vllm/models/qwen4_exp/amd/low_latency_gemm.py deleted file mode 100644 index c6e54885e2..0000000000 --- a/vllm/models/qwen4_exp/amd/low_latency_gemm.py +++ /dev/null @@ -1,14 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Qwen4Exp low-latency GEMM hook for AMD ROCm.""" - -import torch -from torch import nn - - -def enable_qwen4_exp_low_latency_gemm( - module: nn.Module, - dtype: torch.dtype, -) -> None: - """Keep the standard vLLM linear methods on AMD ROCm.""" - del module, dtype diff --git a/vllm/models/qwen4_exp/amd/model.py b/vllm/models/qwen4_exp/amd/model.py deleted file mode 100644 index e7144693d7..0000000000 --- a/vllm/models/qwen4_exp/amd/model.py +++ /dev/null @@ -1,1097 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Inference-only Qwen4Exp model.""" - -from collections.abc import Iterable -from itertools import islice - -import torch -from torch import nn -from transformers import Qwen4ExpConfig, Qwen4ExpTextConfig - -from vllm.compilation.decorators import support_torch_compile -from vllm.config import VllmConfig -from vllm.distributed import get_pp_group -from vllm.model_executor.layers.fused_moe.utils import ( - is_model_fused_shared_expert_compatible, -) -from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm -from vllm.model_executor.layers.logits_processor import LogitsProcessor -from vllm.model_executor.layers.mamba.gdn.qwen_gdn_linear_attn import ( - QwenGatedDeltaNetAttention, -) -from vllm.model_executor.layers.mamba.mamba_utils import ( - MambaStateCopyFunc, - MambaStateCopyFuncCalculator, - MambaStateCopyFuncsByType, - MambaStateDtypeCalculator, - MambaStateShapeCalculator, -) -from vllm.model_executor.layers.quantization import QuantizationConfig -from vllm.model_executor.layers.vocab_parallel_embedding import ( - ParallelLMHead, - VocabParallelEmbedding, -) -from vllm.model_executor.models.interfaces import ( - HasInnerState, - IsHybrid, - MixtureOfExperts, - MultiModalEmbeddings, - SupportsHCSP, - SupportsLoRA, - SupportsMRoPE, - SupportsPP, - _require_is_multimodal, -) -from vllm.model_executor.models.qwen3_5 import ( - Qwen3_5ForConditionalGeneration, - Qwen3_5Model, -) -from vllm.model_executor.models.qwen3_next import ( - Qwen3NextAttention, - Qwen3NextSparseMoeBlock, -) -from vllm.model_executor.models.qwen3_vl import ( - Qwen3_VisionTransformer, - Qwen3VLDummyInputsBuilder, - Qwen3VLMultiModalProcessor, - Qwen3VLProcessingInfo, -) -from vllm.model_executor.models.utils import ( - AutoWeightsLoader, - StageMissingLayer, - WeightsMapper, - _merge_multimodal_embeddings, - extract_layer_index, - make_empty_intermediate_tensors_factory, - make_layers, - maybe_fuse_shared_experts, - maybe_prefix, -) -from vllm.models.common.ops.sequence_parallel import sp_all_gather -from vllm.multimodal import MULTIMODAL_REGISTRY -from vllm.multimodal.inputs import MultiModalFeatureSpec -from vllm.sequence import IntermediateTensors -from vllm.tokenizers.registry import cached_tokenizer_from_config -from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum -from vllm.v1.kv_cache_interface import MambaSpec - -from ..common.hyperconnection import is_hc_sequence_parallel_enabled -from ..common.mtp import make_mtp_hidden_buffer -from .hyperconnection import GatedResidual, HyperConnectionConfig -from .low_latency_gemm import enable_qwen4_exp_low_latency_gemm -from .ops.sequence_parallel import sp_reduce_scatter, sp_shard -from .ple_layer import Qwen4ExpPLELayer -from .qsa import Qwen4ExpQSAAttention - -# Transformers v5.18 renamed `qwen_sparse_attention` to `indexed_attention` -# TODO: Delete qwen_... once Transformers 5.18.0 is the minimum required version. -_QSA_LAYER_TYPES = ("qwen_sparse_attention", "indexed_attention") - - -def without_modelopt_fp4( - quant_config: QuantizationConfig | None, -) -> QuantizationConfig | None: - """Return ``None`` for weights excluded from Qwen4Exp ModelOpt-FP4.""" - if quant_config is not None and quant_config.get_name() == "modelopt_fp4": - return None - return quant_config - - -def _remap_qsa_cache_scale_name( - name: str, - qsa_layer_ids: frozenset[int], -) -> str: - """Map serialized main-cache scales onto the merged QSA owner. - - Regular attention keeps cache scales below its ``attn`` child. QSA owns - that cache directly, so only QSA layers need the final path component - moved to the owner's persistent ``_k_scale``/``_v_scale`` buffers. - """ - scale_suffixes = { - "k_proj.k_scale": "_k_scale", - "k_proj.output_scale": "_k_scale", - "attn.k_scale": "_k_scale", - "attn._k_scale": "_k_scale", - "k_scale": "_k_scale", - "_k_scale": "_k_scale", - "v_proj.v_scale": "_v_scale", - "v_proj.output_scale": "_v_scale", - "attn.v_scale": "_v_scale", - "attn._v_scale": "_v_scale", - "v_scale": "_v_scale", - "_v_scale": "_v_scale", - } - for layer_id in qsa_layer_ids: - marker = f"layers.{layer_id}.self_attn." - marker_start = name.find(marker) - if marker_start < 0 or (marker_start > 0 and name[marker_start - 1] != "."): - continue - suffix = name[marker_start + len(marker) :] - mapped_suffix = scale_suffixes.get(suffix) - if mapped_suffix is not None: - return f"{name[: marker_start + len(marker)]}{mapped_suffix}" - return name - - -_QWEN4_EXP_IGNORED_MISSING_SUFFIXES = [ - ".bias", - "_bias", - ".k_scale", - "_k_scale", - ".v_scale", - "_v_scale", - "_weight_scale", - "_input_scale", -] - -# The checkpoint stores these projections separately; runtime packs each group -# into adjacent logical shards of a MergedColumnParallelLinear. -_EXTRA_WEIGHTS_MAPPER = WeightsMapper( - orig_to_new_stacked={ - "hyper_connection.input_mix_weight_down.weight": ( - "hyper_connection.input_mix_weight_down_block_inject.weight", - 0, - ), - "hyper_connection.block_inject_weight.weight": ( - "hyper_connection.input_mix_weight_down_block_inject.weight", - 1, - ), - "ple.key_proj": ("ple.kv_proj", 0), - "ple.value_proj": ("ple.kv_proj", 1), - } -) - - -class Qwen4ExpSparseMoeBlock(Qwen3NextSparseMoeBlock): - """Qwen3Next MoE with Qwen4Exp HC validation.""" - - def __init__( - self, - vllm_config: VllmConfig, - prefix: str = "", - reduce_results: bool = True, - ) -> None: - super().__init__( - vllm_config=vllm_config, prefix=prefix, reduce_results=reduce_results - ) - if self.replicate_shared_expert and not self.is_sequence_parallel: - # Reduce routed outputs before adding replicated shared outputs. - # SP then slices the result to avoid summing shared outputs again via RS. - self.experts.moe_config.skip_final_all_reduce = False - config = vllm_config.model_config.hf_text_config - self.n_shared_experts = int(config.shared_expert_intermediate_size > 0) - - -class Qwen4ExpDecoderLayer(nn.Module): - def __init__( - self, - vllm_config: VllmConfig, - layer_type: str, - prefix: str = "", - ) -> None: - super().__init__() - config: Qwen4ExpTextConfig = vllm_config.model_config.hf_text_config - quant_config = vllm_config.quant_config - - self.config = config - self.layer_type = layer_type - # MoE SP also enables HC SP, while HC SP can be enabled independently. - self.use_hc_sequence_parallel = is_hc_sequence_parallel_enabled(vllm_config) - self.layer_idx = extract_layer_index(prefix) - self.use_sequence_parallel = ( - vllm_config.parallel_config.use_sequence_parallel_moe - ) - self.ple: Qwen4ExpPLELayer | None = None - ple_layer_ids = config.ple_layer_ids - if (self.layer_idx + 1) in ple_layer_ids: - ple_layer_ids_sorted = sorted(set(ple_layer_ids)) - ple_dense_layer_id_map = { - abs_id: idx for idx, abs_id in enumerate(ple_layer_ids_sorted) - } - ple_dense_layer_id = ple_dense_layer_id_map[self.layer_idx + 1] - self.ple = Qwen4ExpPLELayer( - config, - vllm_config=vllm_config, - layer_idx=self.layer_idx, - ple_dense_layer_id=ple_dense_layer_id, - prefix=f"{prefix}.ple", - use_sequence_parallel=self.use_hc_sequence_parallel, - ) - - if layer_type == "linear_attention": - self.linear_attn = QwenGatedDeltaNetAttention( - config, - vllm_config=vllm_config, - prefix=f"{prefix}.linear_attn", - gqa_interleaved_layout=False, - reduce_results=not self.use_hc_sequence_parallel, - ) - elif layer_type in _QSA_LAYER_TYPES: - use_qsa = getattr(config, "indexer_n_heads", None) is not None - if not use_qsa: - self.self_attn = Qwen3NextAttention( - config, - model_config=vllm_config.model_config, - cache_config=vllm_config.cache_config, - quant_config=quant_config, - prefix=f"{prefix}.self_attn", - reduce_results=not self.use_hc_sequence_parallel, - ) - else: - self.self_attn = Qwen4ExpQSAAttention( - vllm_config=vllm_config, - config=config, - layer_id=self.layer_idx, - quant_config=quant_config, - prefix=f"{prefix}.self_attn", - reduce_results=not self.use_hc_sequence_parallel, - ) - else: - raise ValueError(f"Invalid layer_type {layer_type}") - - self.mlp = Qwen4ExpSparseMoeBlock( - vllm_config=vllm_config, - prefix=f"{prefix}.mlp", - reduce_results=not self.use_hc_sequence_parallel, - ) - - hc_config = HyperConnectionConfig( - hc_count=config.hc_count, - hidden_size=config.hidden_size, - params_dtype=torch.bfloat16, - hc_lowrank=config.hc_lowrank, - rms_norm_eps=config.rms_norm_eps, - hc_per_branch_norm=True, - ) - self.attn_hyper_connection = GatedResidual( - hc_config, - prefix=maybe_prefix(prefix, "attn_hyper_connection"), - ) - self.mlp_hyper_connection = GatedResidual( - hc_config, - prefix=maybe_prefix(prefix, "mlp_hyper_connection"), - ) - - def forward( - self, - hidden_states: torch.Tensor, - prev_block_output: torch.Tensor | None, - prev_injection: torch.Tensor | None, - positions: torch.Tensor, - *, - input_ids: torch.Tensor | None, - query_start_loc: torch.Tensor | None, - ngram_context: torch.Tensor | None, - ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - attn_hc = self.attn_hyper_connection - if self.ple is not None: - # PLE adds directly to the multi-stream state, so pending HC state - # must be materialized before the addition. - if prev_block_output is not None and prev_injection is not None: - hidden_states = attn_hc.combine( - hidden_states, prev_block_output, prev_injection - ) - prev_block_output = prev_injection = None - - if input_ids is None or query_start_loc is None or ngram_context is None: - raise RuntimeError("PLE inputs were not prepared") - hidden_states = self.ple( - hidden_states, - input_ids, - query_start_loc, - ngram_context, - ) - - # Fuse a pending combine with this HC module's mix when possible. - if prev_block_output is not None and prev_injection is not None: - hidden_states, block_input, injection = attn_hc.combine_and_mix( - hidden_states, prev_block_output, prev_injection - ) - else: - hidden_states, block_input, injection = attn_hc.mix(hidden_states) - - if self.use_hc_sequence_parallel: - # Positions retain the full token axis for both text and MRoPE inputs. - block_input = sp_all_gather(block_input)[: positions.shape[-1]] - if self.layer_type == "linear_attention": - attn_out = self.linear_attn(hidden_states=block_input) - elif self.layer_type in _QSA_LAYER_TYPES: - attn_out = self.self_attn( - hidden_states=block_input, - positions=positions, - ) - else: - raise ValueError("Invalid layer_type") - - mlp_hc = self.mlp_hyper_connection - if self.use_hc_sequence_parallel: - attn_out = sp_reduce_scatter(attn_out) - hidden_states, block_input, injection = mlp_hc.combine_and_mix( - hidden_states, attn_out, injection - ) - if self.use_sequence_parallel: - # MoE consumes local tokens and returns complete local outputs. - mlp_out = self.mlp(block_input, already_sequence_parallel=True) - else: - if self.use_hc_sequence_parallel: - block_input = sp_all_gather(block_input)[: positions.shape[-1]] - mlp_out = self.mlp(block_input) - if self.use_hc_sequence_parallel: - assert isinstance(self.mlp, Qwen4ExpSparseMoeBlock) - reduce_output = self.mlp.experts.moe_config.skip_final_all_reduce - # Only sum partial outputs. Reduced outputs already include all ranks. - mlp_out = ( - sp_reduce_scatter(mlp_out) if reduce_output else sp_shard(mlp_out) - ) - return hidden_states, mlp_out, injection - - -class Qwen4ExpMixtureOfExperts(MixtureOfExperts): - """Expose Qwen4Exp routed experts through vLLM's EPLB protocol.""" - - def set_moe_parameters(self, layers: Iterable[nn.Module]) -> None: - self.moe_layers = [] - self.moe_mlp_layers = [] - example_moe = None - for layer in layers: - if isinstance(layer, Qwen4ExpDecoderLayer) and isinstance( - layer.mlp, Qwen4ExpSparseMoeBlock - ): - example_moe = layer.mlp - self.moe_mlp_layers.append(layer.mlp) - self.moe_layers.append(layer.mlp.experts) - - self.num_moe_layers = len(self.moe_layers) - if example_moe is None: - self.num_expert_groups = 0 - self.num_shared_experts = 0 - self.num_logical_experts = 0 - self.num_physical_experts = 0 - self.num_local_physical_experts = 0 - self.num_routed_experts = 0 - self.num_redundant_experts = 0 - return - - self.num_expert_groups = 1 - self.num_shared_experts = example_moe.n_shared_experts - self.num_logical_experts = example_moe.n_logical_experts - self.num_physical_experts = example_moe.n_physical_experts - self.num_local_physical_experts = example_moe.n_local_physical_experts - self.num_routed_experts = example_moe.n_routed_experts - self.num_redundant_experts = example_moe.n_redundant_experts - - def update_physical_experts_metadata( - self, - num_physical_experts: int, - num_local_physical_experts: int, - ) -> None: - assert self.num_local_physical_experts == num_local_physical_experts - self.num_physical_experts = num_physical_experts - self.num_local_physical_experts = num_local_physical_experts - self.num_redundant_experts = num_physical_experts - self.num_logical_experts - for moe in self.moe_mlp_layers: - moe.n_physical_experts = num_physical_experts - moe.n_local_physical_experts = num_local_physical_experts - moe.n_redundant_experts = self.num_redundant_experts - moe.experts.update_expert_map() - - -@support_torch_compile( - dynamic_arg_dims={ - "input_ids": 0, - "positions": -1, - "intermediate_tensors": 0, - "inputs_embeds": 0, - "query_start_loc": 0, - "ngram_context": 0, - "deepstack_input_embeds": 0, - } -) -class Qwen4ExpModel(nn.Module): - hf_to_vllm_mapper = Qwen3_5Model.hf_to_vllm_mapper | _EXTRA_WEIGHTS_MAPPER - - def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None: - super().__init__() - config: Qwen4ExpTextConfig = vllm_config.model_config.hf_text_config - self.config = config - self.num_redundant_experts = ( - vllm_config.parallel_config.eplb_config.num_redundant_experts - ) - # MoE SP also enables HC SP, while HC SP can be enabled independently. - self.use_hc_sequence_parallel = is_hc_sequence_parallel_enabled(vllm_config) - self.vocab_size = config.vocab_size - self._qsa_layer_ids = frozenset( - layer_idx - for layer_idx, layer_type in enumerate(config.layer_types) - if layer_type in _QSA_LAYER_TYPES - and getattr(config, "indexer_n_heads", None) is not None - ) - self.embed_tokens = VocabParallelEmbedding(self.vocab_size, config.hidden_size) - - def get_layer(prefix: str) -> Qwen4ExpDecoderLayer: - layer_idx = extract_layer_index(prefix) - return Qwen4ExpDecoderLayer( - vllm_config, - layer_type=config.layer_types[layer_idx], - prefix=prefix, - ) - - self.start_layer, self.end_layer, self.layers = make_layers( - config.num_hidden_layers, get_layer, prefix=f"{prefix}.layers" - ) - self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible( - self.layers, - Qwen4ExpSparseMoeBlock, - "mlp", - ) - intermediate_size = config.hidden_size * config.hc_count - self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory( - ["hidden_states"], intermediate_size - ) - - self.hyper_connection_mixer: GatedResidual | None - if get_pp_group().is_last_rank: - hc_config = HyperConnectionConfig( - hc_count=config.hc_count, - hidden_size=config.hidden_size, - params_dtype=torch.bfloat16, - hc_lowrank=config.hc_lowrank, - rms_norm_eps=config.rms_norm_eps, - hc_per_branch_norm=True, - ) - self.hyper_connection_mixer = GatedResidual( - hc_config, - use_combine=False, - prefix=maybe_prefix(prefix, "hyper_connection_mixer"), - ) - else: - self.hyper_connection_mixer = None - - self._mtp_hidden_buffer = make_mtp_hidden_buffer( - vllm_config, - config, - is_last_rank=get_pp_group().is_last_rank, - ) - - def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: - return self.embed_tokens(input_ids) - - def forward( - self, - input_ids: torch.Tensor | None, - positions: torch.Tensor, - intermediate_tensors: IntermediateTensors | None = None, - inputs_embeds: torch.Tensor | None = None, - query_start_loc: torch.Tensor | None = None, - ngram_context: torch.Tensor | None = None, - deepstack_input_embeds: IntermediateTensors | None = None, - ) -> torch.Tensor | IntermediateTensors: - if get_pp_group().is_first_rank: - if inputs_embeds is not None: - hidden_states = inputs_embeds - else: - if input_ids is None: - raise ValueError("input_ids or inputs_embeds is required") - hidden_states = self.embed_input_ids(input_ids) - if self.use_hc_sequence_parallel: - # The MoE padding mask stays unsharded: under torch.compile its - # update would land only after the graph has run. - hidden_states = sp_shard(hidden_states) - # Expand only the local token rows when SP is enabled. - hidden_states = hidden_states.repeat(1, self.config.hc_count) - else: - if intermediate_tensors is None: - raise ValueError("pipeline stage requires intermediate tensors") - hidden_states = intermediate_tensors["hidden_states"] - - full_num_tokens = positions.shape[-1] - block_output = None - injection = None - last_layer = None - for layer_idx, layer in islice( - enumerate(self.layers), self.start_layer, self.end_layer - ): - last_layer = layer - hidden_states, block_output, injection = layer( - hidden_states=hidden_states, - prev_block_output=block_output, - prev_injection=injection, - positions=positions, - input_ids=input_ids, - query_start_loc=query_start_loc, - ngram_context=ngram_context, - ) - if deepstack_input_embeds is not None and layer_idx < len( - deepstack_input_embeds - ): - deepstack_embed = deepstack_input_embeds[ - f"deepstack_input_embeds_{layer_idx}" - ] - if self.use_hc_sequence_parallel: - deepstack_embed = sp_shard(deepstack_embed) - deepstack_embed = ( - deepstack_embed.unsqueeze(-2) - .expand( - *deepstack_embed.shape[:-1], - self.config.hc_count, - self.config.hidden_size, - ) - .flatten(-2) - ) - # Deepstack is an external addition to the materialized - # multi-stream state and therefore terminates delayed combine. - hidden_states = layer.mlp_hyper_connection.combine( - hidden_states, block_output, injection - ) - block_output = None - injection = None - hidden_states = hidden_states + deepstack_embed - - if not get_pp_group().is_last_rank: - # PP transports one tensor, not the delayed HC tuple. Materialize - # with the HC module that produced the pending injection. - if last_layer is not None and block_output is not None: - hidden_states = last_layer.mlp_hyper_connection.combine( - hidden_states, block_output, injection - ) - return IntermediateTensors({"hidden_states": hidden_states}) - - # The final mixer consumes the last pending combine and returns both - # the sampled single stream and the materialized multi-stream state. - final_mixer = self.hyper_connection_mixer - assert final_mixer is not None - multi_hidden, sample_hidden_states, _ = final_mixer.combine_and_mix( - hidden_states, block_output, injection - ) - if self.use_hc_sequence_parallel: - if self._mtp_hidden_buffer is not None: - # Gather LM-head and MTP states together when both are needed. - hidden_size = self.config.hidden_size - packed_hidden_states = torch.cat( - [sample_hidden_states, multi_hidden], dim=-1 - ) - packed_hidden_states = sp_all_gather(packed_hidden_states)[ - :full_num_tokens - ] - sample_hidden_states, multi_hidden = packed_hidden_states.split( - [hidden_size, self.config.hc_count * hidden_size], dim=-1 - ) - sample_hidden_states = sample_hidden_states.contiguous() - else: - sample_hidden_states = sp_all_gather(sample_hidden_states)[ - :full_num_tokens - ] - if self._mtp_hidden_buffer is not None: - # Capture the pre-final-mixer multi-stream hidden state - # [T, hc_count*H] for the MTP drafter (zero extra compute: - # this tensor is needed by the final mixer regardless). - num_tokens = multi_hidden.shape[0] - self._mtp_hidden_buffer[:num_tokens].copy_(multi_hidden) - return sample_hidden_states - - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - weights = ( - ( - _remap_qsa_cache_scale_name(name, self._qsa_layer_ids), - weight, - ) - for name, weight in weights - ) - weights = maybe_fuse_shared_experts( - weights, - enabled=self.is_fused_shared_expert_enabled, - n_routed_experts=getattr(self.config, "num_experts", 0) or 0, - n_shared_experts=1, - ckpt_prefix="mlp.shared_expert", - ) - # Non-persistent PLE state rebuilt in __init__; skip any ckpt - # column for them. - skip_substrs = ( - "hashstats_", - "token_lookup", - "hyper_connection_mixer.block_inject_weight", - ) - mapper = self.hf_to_vllm_mapper | WeightsMapper( - orig_to_new_substr={substr: None for substr in skip_substrs} - ) - # The final HC mixer only exists on the last PP rank; earlier ranks - # must drop its checkpoint weights instead of failing to place them. - ignore_prefixes = ( - None - if self.hyper_connection_mixer is not None - else ["hyper_connection_mixer."] - ) - loader = AutoWeightsLoader( - self, - ignore_unexpected_prefixes=ignore_prefixes, - ignore_unexpected_suffixes=_QWEN4_EXP_IGNORED_MISSING_SUFFIXES.copy(), - ) - loaded = loader.load_weights( - weights, - mapper=mapper, - ) - return loaded - - -class Qwen4ExpForCausalLM( - nn.Module, - HasInnerState, - SupportsHCSP, - SupportsLoRA, - SupportsMRoPE, - SupportsPP, - Qwen4ExpMixtureOfExperts, - IsHybrid, -): - packed_modules_mapping = { - "qkv_proj": ["q_proj", "k_proj", "v_proj"], - "gate_up_proj": ["gate_proj", "up_proj"], - "kv_proj": ["key_proj", "value_proj"], - "in_proj_qkvz": ["in_proj_qkv", "in_proj_z"], - "in_proj_ba": ["in_proj_b", "in_proj_a"], - "input_mix_weight_down_block_inject": [ - "input_mix_weight_down", - "block_inject_weight", - "_input_mix_padding", - ], - } - hf_to_vllm_mapper = WeightsMapper( - orig_to_new_prefix={"model.language_model.": "model."} - ) - requires_raw_input_tokens = True - - def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None: - super().__init__() - config: Qwen4ExpTextConfig = vllm_config.model_config.hf_text_config - self.vllm_config = vllm_config - self.model_config = vllm_config.model_config - self.quant_config = vllm_config.quant_config - self.config = config - self.scheduler_config = vllm_config.scheduler_config - self.model = Qwen4ExpModel( - vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model") - ) - self.lm_head = ParallelLMHead( - config.vocab_size, - config.hidden_size, - prefix=maybe_prefix(prefix, "lm_head"), - ) - self.logits_processor = LogitsProcessor(config.vocab_size) - self.make_empty_intermediate_tensors = ( - self.model.make_empty_intermediate_tensors - ) - self.set_moe_parameters(self.model.layers) - enable_qwen4_exp_low_latency_gemm(self, self.model_config.dtype) - - @staticmethod - def get_model_state_cls(): - from .model_state import Qwen4ExpModelState - - return Qwen4ExpModelState - - def forward( - self, - input_ids: torch.Tensor | None, - positions: torch.Tensor, - intermediate_tensors: IntermediateTensors | None = None, - inputs_embeds: torch.Tensor | None = None, - **kwargs: object, - ) -> torch.Tensor | IntermediateTensors: - return self.model( - input_ids, - positions, - intermediate_tensors, - inputs_embeds, - **kwargs, - ) - - def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: - return self.model.embed_input_ids(input_ids) - - @classmethod - def get_ple_mamba_state_dtype_from_config( - cls, - vllm_config: VllmConfig, - ) -> tuple[torch.dtype, ...]: - return MambaStateDtypeCalculator.short_conv_state_dtype( - vllm_config.model_config.dtype, - vllm_config.cache_config.mamba_cache_dtype, - ) - - @classmethod - def get_ple_mamba_state_shape_from_config( - cls, vllm_config: VllmConfig - ) -> tuple[tuple[int, int]]: - hf_config = vllm_config.model_config.hf_text_config - conv_kernel_size = hf_config.ple_conv_kernel_size - short_conv_dilation = hf_config.ngram_size - conv_state_len = (conv_kernel_size - 1) * short_conv_dilation - num_spec = ( - vllm_config.speculative_config.num_speculative_tokens - if vllm_config.speculative_config - else 0 - ) - hc_count = hf_config.hc_count - hc_hidden_size = hf_config.hidden_size * hc_count - return MambaStateShapeCalculator.short_conv_state_shape( - tp_world_size=1, - intermediate_size=hc_hidden_size, - conv_kernel=conv_state_len + 1, - num_spec=num_spec, - ) - - @classmethod - def get_gdn_mamba_state_dtype_from_config( - cls, vllm_config: VllmConfig - ) -> tuple[torch.dtype, torch.dtype]: - return MambaStateDtypeCalculator.gated_delta_net_state_dtype( - vllm_config.model_config.dtype, - vllm_config.cache_config.mamba_cache_dtype, - vllm_config.cache_config.mamba_ssm_cache_dtype, - ) - - @classmethod - def get_gdn_mamba_state_shape_from_config( - cls, vllm_config: VllmConfig - ) -> tuple[tuple[int, int], tuple[int, int]]: - parallel_config = vllm_config.parallel_config - hf_config = vllm_config.model_config.hf_text_config - tp_size = parallel_config.tensor_parallel_size - num_spec = ( - vllm_config.speculative_config.num_speculative_tokens - if vllm_config.speculative_config - else 0 - ) - return MambaStateShapeCalculator.gated_delta_net_state_shape( - tp_size, - hf_config.linear_num_key_heads, - hf_config.linear_num_value_heads, - hf_config.linear_key_head_dim, - hf_config.linear_value_head_dim, - hf_config.linear_conv_kernel_dim, - num_spec, - ) - - @classmethod - def get_mamba_state_dtype_from_config( - cls, - vllm_config: VllmConfig, - ) -> tuple[torch.dtype, torch.dtype]: - return cls.get_gdn_mamba_state_dtype_from_config(vllm_config) - - @classmethod - def get_mamba_state_shape_from_config( - cls, vllm_config: VllmConfig - ) -> tuple[tuple[int, int], tuple[int, int]]: - return cls.get_gdn_mamba_state_shape_from_config(vllm_config) - - @classmethod - def get_mamba_state_copy_func( - cls, - ) -> tuple[MambaStateCopyFunc, MambaStateCopyFunc]: - return MambaStateCopyFuncCalculator.gated_delta_net_state_copy_func() - - @classmethod - def get_mamba_state_copy_funcs( - cls, - mamba_types: set[MambaAttentionBackendEnum], - ) -> MambaStateCopyFuncsByType: - copy_funcs_by_type = { - MambaAttentionBackendEnum.GDN_ATTN: cls.get_mamba_state_copy_func(), - MambaAttentionBackendEnum.SHORT_CONV: ( - MambaStateCopyFuncCalculator.short_conv_state_copy_func() - ), - } - missing_types = mamba_types - copy_funcs_by_type.keys() - assert not missing_types, f"missing state copy funcs for {missing_types}" - return { - mamba_type: copy_funcs_by_type[mamba_type] for mamba_type in mamba_types - } - - @classmethod - def get_mamba_specs_from_config( - cls, vllm_config: VllmConfig - ) -> tuple[MambaSpec, ...]: - """Return all MambaSpecs for this model (GDN layers + PLE layer). - - The PLE layer uses a separate short_conv MambaSpec whose page_size_bytes - may exceed the GDN spec; callers should take the maximum. - """ - return ( - MambaSpec( - shapes=cls.get_gdn_mamba_state_shape_from_config(vllm_config), - dtypes=cls.get_gdn_mamba_state_dtype_from_config(vllm_config), - block_size=-1, - ), - MambaSpec( - shapes=cls.get_ple_mamba_state_shape_from_config(vllm_config), - dtypes=cls.get_ple_mamba_state_dtype_from_config(vllm_config), - block_size=-1, - tp_replicated=True, - ), - ) - - def compute_logits(self, hidden_states: torch.Tensor) -> torch.Tensor | None: - return self.logits_processor(self.lm_head, hidden_states) - - def get_mtp_target_hidden_states(self) -> torch.Tensor | None: - return self.model._mtp_hidden_buffer - - def get_mrope_input_positions( - self, - input_tokens: list[int], - mm_features: list[MultiModalFeatureSpec], - ) -> tuple[torch.Tensor, int]: - positions = torch.arange(len(input_tokens), dtype=torch.long) - return positions.unsqueeze(0).expand(3, -1), 0 - - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - mapper = self.hf_to_vllm_mapper | WeightsMapper( - orig_to_new_substr={"mtp.": None} - ) - loader = AutoWeightsLoader( - self, - ignore_unexpected_suffixes=_QWEN4_EXP_IGNORED_MISSING_SUFFIXES.copy(), - ) - return loader.load_weights(weights, mapper=mapper) - - -class Qwen4ExpProcessingInfo(Qwen3VLProcessingInfo): - def get_hf_config(self) -> Qwen4ExpConfig: - return self.ctx.get_hf_config(Qwen4ExpConfig) - - -@MULTIMODAL_REGISTRY.register_processor( - Qwen3VLMultiModalProcessor, - info=Qwen4ExpProcessingInfo, - dummy_inputs=Qwen3VLDummyInputsBuilder, -) -class Qwen4ExpForConditionalGeneration( - Qwen3_5ForConditionalGeneration, - HasInnerState, - SupportsHCSP, - Qwen4ExpMixtureOfExperts, -): - """Qwen3-VL vision tower backed by the Qwen4Exp language model.""" - - requires_raw_input_tokens = True - - packed_modules_mapping = Qwen3_5ForConditionalGeneration.packed_modules_mapping | { - "kv_proj": ["key_proj", "value_proj"], - "input_mix_weight_down_block_inject": [ - "input_mix_weight_down", - "block_inject_weight", - "_input_mix_padding", - ], - } - - @staticmethod - def get_model_state_cls(): - from .model_state import Qwen4ExpModelState - - return Qwen4ExpModelState - - def __init__(self, *, vllm_config: VllmConfig, prefix: str = "model") -> None: - nn.Module.__init__(self) - config: Qwen4ExpConfig = vllm_config.model_config.hf_config - quant_config = vllm_config.quant_config - multimodal_config = vllm_config.model_config.multimodal_config - if multimodal_config is None: - raise ValueError( - "Qwen4ExpForConditionalGeneration requires multimodal_config" - ) - - self.config = config - self.model_config = vllm_config.model_config - self.multimodal_config = multimodal_config - self.language_model_only = multimodal_config.language_model_only - if self.language_model_only: - self.use_data_parallel = False - self.is_multimodal_pruning_enabled = False - self.video_pruning_method = None - self.video_pruning_rate = 0.0 - self._tokenizer = None - self.visual = StageMissingLayer("vision_tower") - self._tower_model_names = [] - else: - self.use_data_parallel = multimodal_config.mm_encoder_tp_mode == "data" - self._init_video_pruning(multimodal_config) - self._tokenizer = cached_tokenizer_from_config(vllm_config.model_config) - - with self._mark_tower_model(vllm_config, {"image", "video"}): - self.visual = Qwen3_VisionTransformer( - config.vision_config, - norm_eps=config.text_config.rms_norm_eps, - quant_config=quant_config, - input_norm=build_mm_input_norm(self.model_config), - prefix=maybe_prefix(prefix, "visual"), - ) - - self.use_deepstack = ( - not self.language_model_only - and bool(config.vision_config.deepstack_visual_indexes) - and not isinstance(self.visual, StageMissingLayer) - ) - self.deepstack_num_level = ( - len(config.vision_config.deepstack_visual_indexes) - if self.use_deepstack - else 0 - ) - self.visual_dim = config.vision_config.out_hidden_size - self.multiscale_dim = self.visual_dim * self.deepstack_num_level - - if self.use_deepstack: - self.deepstack_input_embeds = [ - torch.zeros( - vllm_config.scheduler_config.max_num_batched_tokens, - config.text_config.hidden_size, - ) - for _ in range(self.deepstack_num_level) - ] - self.deepstack_input_embeds_num_tokens = 0 - - with self._mark_language_model(vllm_config): - self.language_model = Qwen4ExpForCausalLM( - vllm_config=vllm_config, - prefix=maybe_prefix(prefix, "language_model"), - ) - - self.make_empty_intermediate_tensors = ( - self.language_model.make_empty_intermediate_tensors - ) - if not get_pp_group().is_first_rank and self.use_deepstack: - assert self.language_model.model.start_layer >= len( - config.vision_config.deepstack_visual_indexes - ), ( - "start_layer should be greater than or equal to " - "len(deepstack_visual_indexes)" - ) - self.set_moe_parameters(self.language_model.model.layers) - - def embed_input_ids( - self, - input_ids: torch.Tensor, - multimodal_embeddings: MultiModalEmbeddings | None = None, - *, - is_multimodal: torch.Tensor | None = None, - ) -> torch.Tensor: - inputs_embeds = self._embed_text_input_ids( - input_ids, - self.language_model.embed_input_ids, - is_multimodal=is_multimodal, - ) - if multimodal_embeddings is None or len(multimodal_embeddings) == 0: - return inputs_embeds - if self.language_model_only: - raise ValueError( - "Qwen4Exp language_model_only does not accept multimodal embeddings" - ) - - is_multimodal = _require_is_multimodal(is_multimodal) - if self.use_deepstack: - deepstack_input_embeds, multimodal_embeddings = ( - self._compute_deepstack_embeds( - inputs_embeds=inputs_embeds, - multimodal_embeddings=multimodal_embeddings, - is_multimodal=is_multimodal, - ) - ) - else: - deepstack_input_embeds = None - - inputs_embeds = _merge_multimodal_embeddings( - inputs_embeds=inputs_embeds, - multimodal_embeddings=multimodal_embeddings, - is_multimodal=is_multimodal, - ) - if deepstack_input_embeds is not None: - self._set_deepstack_input_embeds(deepstack_input_embeds) - return inputs_embeds - - def get_mtp_target_hidden_states(self) -> torch.Tensor | None: - return self.language_model.get_mtp_target_hidden_states() - - def forward( - self, - input_ids: torch.Tensor | None, - positions: torch.Tensor, - intermediate_tensors: IntermediateTensors | None = None, - inputs_embeds: torch.Tensor | None = None, - **kwargs: object, - ) -> torch.Tensor | IntermediateTensors: - if intermediate_tensors is not None: - inputs_embeds = None - if inputs_embeds is not None and get_pp_group().is_first_rank: - deepstack_input_embeds = self._get_deepstack_input_embeds( - inputs_embeds.size(0) - ) - else: - deepstack_input_embeds = None - - hidden_states = self.language_model.model( - input_ids=input_ids, - positions=positions, - intermediate_tensors=intermediate_tensors, - inputs_embeds=inputs_embeds, - query_start_loc=kwargs.get("query_start_loc"), - ngram_context=kwargs.get("ngram_context"), - deepstack_input_embeds=deepstack_input_embeds, - ) - if inputs_embeds is not None and get_pp_group().is_first_rank: - self._clear_deepstack_input_embeds(inputs_embeds.size(0)) - return hidden_states - - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - mapper = self.hf_to_vllm_mapper | WeightsMapper( - orig_to_new_substr={"mtp.": None}, - orig_to_new_prefix={"visual.": None} if self.language_model_only else {}, - ) - loader = AutoWeightsLoader( - self, - ignore_unexpected_suffixes=_QWEN4_EXP_IGNORED_MISSING_SUFFIXES.copy(), - ) - return loader.load_weights(weights, mapper=mapper) - - @classmethod - def get_mamba_state_dtype_from_config( - cls, - vllm_config: VllmConfig, - ) -> tuple[torch.dtype, torch.dtype]: - return Qwen4ExpForCausalLM.get_mamba_state_dtype_from_config(vllm_config) - - @classmethod - def get_mamba_state_shape_from_config( - cls, - vllm_config: VllmConfig, - ) -> tuple[tuple[int, int], tuple[int, int]]: - return Qwen4ExpForCausalLM.get_mamba_state_shape_from_config(vllm_config) - - @classmethod - def get_mamba_state_copy_func( - cls, - ) -> tuple[MambaStateCopyFunc, MambaStateCopyFunc]: - return Qwen4ExpForCausalLM.get_mamba_state_copy_func() - - @classmethod - def get_mamba_state_copy_funcs( - cls, - mamba_types: set[MambaAttentionBackendEnum], - ) -> MambaStateCopyFuncsByType: - return Qwen4ExpForCausalLM.get_mamba_state_copy_funcs(mamba_types) - - @classmethod - def get_mamba_specs_from_config( - cls, vllm_config: VllmConfig - ) -> tuple[MambaSpec, ...]: - return Qwen4ExpForCausalLM.get_mamba_specs_from_config(vllm_config) - - -__all__ = [ - "Qwen4ExpDecoderLayer", - "Qwen4ExpForCausalLM", - "Qwen4ExpForConditionalGeneration", - "Qwen4ExpMixtureOfExperts", - "Qwen4ExpModel", - "Qwen4ExpSparseMoeBlock", -] diff --git a/vllm/models/qwen4_exp/amd/model_state.py b/vllm/models/qwen4_exp/amd/model_state.py deleted file mode 100644 index ec2a0fef8b..0000000000 --- a/vllm/models/qwen4_exp/amd/model_state.py +++ /dev/null @@ -1,143 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Model-runner state for Qwen4Exp PLE inputs.""" - -from typing import Any - -import torch -import torch.nn as nn - -from vllm.config import VllmConfig -from vllm.v1.worker.gpu.input_batch import InputBatch -from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache -from vllm.v1.worker.gpu.model_states.mamba_hybrid import MambaHybridModelState -from vllm.v1.worker.gpu.states import RequestState - - -class Qwen4ExpModelState(MambaHybridModelState): - """Add rollback-safe PLE n-gram context to the model inputs.""" - - def __init__( - self, - vllm_config: VllmConfig, - model: nn.Module, - encoder_cache: EncoderCache | None, - device: torch.device, - ) -> None: - super().__init__(vllm_config, model, encoder_cache, device) - config = self.model_config.hf_text_config - self.uses_ngram_embedding = bool(config.ple_layer_ids) - if not self.uses_ngram_embedding: - self.ngram_context_len = 0 - self.ngram_eos_token_id = 0 - return - - if vllm_config.parallel_config.pipeline_parallel_size > 1: - raise RuntimeError( - "N-gram PLE embedding currently requires " - "pipeline_parallel_size=1 because non-first pipeline ranks do " - "not receive the raw input_ids required by PLE. Please run " - "with PP=1." - ) - - self.ngram_context_len = int(config.ngram_size) - 1 - if self.ngram_context_len <= 0: - raise ValueError("N-gram embedding requires context length >= 1.") - self.ngram_eos_token_id = int(config.eos_token_id) - self.ngram_context = torch.full( - (self.max_num_reqs, self.ngram_context_len), - self.ngram_eos_token_id, - dtype=torch.int32, - device=self.device, - ) - self.ngram_context_offsets = torch.arange( - -self.ngram_context_len, - 0, - dtype=torch.int64, - device=self.device, - ) - self.ple_query_start_loc = torch.zeros( - self.max_num_reqs + 1, - dtype=torch.int32, - device=self.device, - ) - - def _prepare_ngram_context( - self, - input_batch: InputBatch, - req_states: RequestState, - ) -> torch.Tensor: - num_reqs = input_batch.num_reqs - num_reqs_padded = input_batch.num_reqs_after_padding - context = self.ngram_context[:num_reqs_padded] - context.fill_(self.ngram_eos_token_id) - if num_reqs == 0: - return context - - request_indices = input_batch.idx_mapping[:num_reqs].long() - context_end = req_states.num_computed_tokens.gpu[request_indices].long() - token_indices = context_end.unsqueeze(1) + self.ngram_context_offsets - valid_tokens = token_indices >= 0 - token_indices.clamp_min_(0) - context_tokens = req_states.all_token_ids.gpu[ - request_indices.unsqueeze(1), token_indices - ] - context[:num_reqs].copy_( - torch.where( - valid_tokens, - context_tokens, - context_tokens.new_full((), self.ngram_eos_token_id), - ) - ) - return context - - def prepare_inputs( - self, - input_batch: InputBatch, - req_states: RequestState, - ) -> dict[str, Any]: - model_inputs = super().prepare_inputs(input_batch, req_states) - if not self.uses_ngram_embedding: - return model_inputs - - num_reqs_padded = input_batch.num_reqs_after_padding - query_start_loc = self.ple_query_start_loc[: num_reqs_padded + 1] - query_start_loc.copy_(input_batch.query_start_loc[: num_reqs_padded + 1]) - model_inputs.update( - query_start_loc=query_start_loc, - ngram_context=self._prepare_ngram_context(input_batch, req_states), - ) - return model_inputs - - def prepare_dummy_inputs( - self, - num_reqs: int, - num_tokens: int, - ) -> dict[str, Any]: - model_inputs = super().prepare_dummy_inputs(num_reqs, num_tokens) - if not self.uses_ngram_embedding: - return model_inputs - - query_start_loc = self.ple_query_start_loc[: num_reqs + 1] - query_start_loc[0] = 0 - tokens_per_req, num_extra_tokens = divmod(num_tokens, num_reqs) - query_lens = torch.full( - (num_reqs,), - tokens_per_req, - dtype=query_start_loc.dtype, - device=query_start_loc.device, - ) - if num_extra_tokens > 0: - query_lens[-num_extra_tokens:] += 1 - torch.cumsum(query_lens, dim=0, out=query_start_loc[1:]) - - ngram_context = self.ngram_context[:num_reqs] - ngram_context.fill_(self.ngram_eos_token_id) - model_inputs.update( - query_start_loc=query_start_loc, - ngram_context=ngram_context, - ) - return model_inputs - - -__all__ = ["Qwen4ExpModelState"] diff --git a/vllm/models/qwen4_exp/amd/mtp.py b/vllm/models/qwen4_exp/amd/mtp.py deleted file mode 100644 index 73c9833335..0000000000 --- a/vllm/models/qwen4_exp/amd/mtp.py +++ /dev/null @@ -1,495 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Inference-only Qwen4Exp MTP (Multi-Token Predictor) model. - -The MTP draft model reuses the Qwen4Exp backbone (PLE/HC/MoE) but: - - drops all multi-modal handling (text-only), - - forces PLE off while keeping the main model's HC stream count, - - fuses the backbone hidden and the new-token embedding via - ``residual_linear_shared`` (fc_embedding + shared fc_hidden) instead of - the ``Linear(2H, H)`` + repeat used by other MTP variants, - - emits TWO hidden streams per step (scheme A): a single stream [T, H] - (final-mixer collapsed, fed to the LM head) and a pre-final-mixer - multi stream [T, hc_count*H] (fed to the next draft step). -""" - -from collections.abc import Iterable - -import regex as re -import torch -from torch import nn -from transformers import Qwen4ExpTextConfig - -from vllm.compilation.decorators import support_torch_compile -from vllm.config import VllmConfig, replace, set_current_vllm_config -from vllm.distributed import get_pp_group -from vllm.model_executor.layers.fused_moe.utils import ( - is_model_fused_shared_expert_compatible, -) -from vllm.model_executor.layers.layernorm import GemmaRMSNorm -from vllm.model_executor.layers.linear import ColumnParallelLinear -from vllm.model_executor.layers.logits_processor import LogitsProcessor -from vllm.model_executor.layers.vocab_parallel_embedding import ( - ParallelLMHead, - VocabParallelEmbedding, -) -from vllm.model_executor.model_loader.utils import configure_quant_config -from vllm.model_executor.models.interfaces import ( - LocalArgmaxMixin, - SupportsHCSP, - SupportsPP, -) -from vllm.model_executor.models.qwen3_5 import Qwen3_5Model -from vllm.model_executor.models.utils import ( - AutoWeightsLoader, - PPMissingLayer, - WeightsMapper, - get_draft_quant_config, - make_empty_intermediate_tensors_factory, - maybe_fuse_shared_experts, - maybe_prefix, -) -from vllm.models.common.ops.sequence_parallel import sp_all_gather, sp_shard -from vllm.sequence import IntermediateTensors - -from ..common.hyperconnection import is_hc_sequence_parallel_enabled -from .hyperconnection import GatedResidual, HyperConnectionConfig -from .low_latency_gemm import enable_qwen4_exp_low_latency_gemm -from .model import ( - _EXTRA_WEIGHTS_MAPPER, - _QWEN4_EXP_IGNORED_MISSING_SUFFIXES, - Qwen4ExpDecoderLayer, - Qwen4ExpMixtureOfExperts, - Qwen4ExpSparseMoeBlock, -) - - -def _remap_ignored_layers( - ignored_layers: list[str], - mtp_start_layer_idx: int, -) -> list[str]: - remapped: list[str] = [] - for name in ignored_layers: - if name.startswith("mtp."): - new_name = re.sub( - r"(?<=\.layers\.)\d+", - lambda m: str(mtp_start_layer_idx + int(m.group(0))), - name, - ) - remapped.append(new_name) - else: - remapped.append(name) - return remapped - - -def _remap_quantized_layers( - quantized_layers: dict[str, dict], - mtp_start_layer_idx: int, -) -> dict[str, dict]: - """Map checkpoint MTP layer indices to standalone draft indices.""" - return { - _remap_ignored_layers([name], mtp_start_layer_idx)[0]: layer_info - for name, layer_info in quantized_layers.items() - } - - -def _remap_mtp_weight_name(name: str) -> str | None: - """Map Qwen4Exp checkpoint paths into the standalone draft model.""" - for checkpoint_prefix in ( - "model.language_model.", - "language_model.", - ): - if name.startswith(checkpoint_prefix): - name = name.removeprefix(checkpoint_prefix) - break - - if name.startswith("embed_tokens."): - name = f"model.{name}" - if name.startswith("model.mtp."): - name = name.removeprefix("model.") - if name.startswith("mtp.shared_head.head."): - return name.replace("mtp.shared_head.head.", "lm_head.", 1) - if name.startswith("model.shared_head.head."): - return name.replace("model.shared_head.head.", "lm_head.", 1) - if name.startswith("shared_head.head."): - return name.replace("shared_head.head.", "lm_head.", 1) - if name.startswith("model.lm_head."): - return name.removeprefix("model.") - if name.startswith("mtp."): - return name.replace("mtp.", "model.", 1) - if name.startswith("model.embed_tokens.") or name.startswith("lm_head."): - return name - return None - - -def _make_draft_vllm_config( - vllm_config: VllmConfig, - mtp_start_layer_idx: int, -) -> VllmConfig: - """Ensure that the draft model config is set in the vLLM config.""" - speculative_config = vllm_config.speculative_config - if speculative_config is None or speculative_config.draft_model_config is None: - raise ValueError("speculative_config.draft_model_config must be set") - - draft_quant_config = get_draft_quant_config(vllm_config) - - # inject packed and ignored modules to the quantization config of draft model - if draft_quant_config is not None: - configure_quant_config(draft_quant_config, Qwen4ExpMTP) - ignored_layers = getattr(draft_quant_config, "ignored_layers", None) - if ignored_layers: - setattr( # noqa: B010 - draft_quant_config, - "ignored_layers", - _remap_ignored_layers(ignored_layers, mtp_start_layer_idx), - ) - exclude_modules = getattr(draft_quant_config, "exclude_modules", None) - if exclude_modules: - setattr( # noqa: B010 - draft_quant_config, - "exclude_modules", - _remap_ignored_layers(exclude_modules, mtp_start_layer_idx), - ) - quantized_layers = getattr(draft_quant_config, "quantized_layers", None) - if quantized_layers: - setattr( # noqa: B010 - draft_quant_config, - "quantized_layers", - _remap_quantized_layers(quantized_layers, mtp_start_layer_idx), - ) - - draft_vllm_config = replace( - vllm_config, - model_config=speculative_config.draft_model_config, - ) - # VllmConfig post-init derives the target quant config, so restore the - # independently resolved draft quant config after replacement. - draft_vllm_config.quant_config = draft_quant_config - return draft_vllm_config - - -@support_torch_compile( - dynamic_arg_dims={ - "input_ids": 0, - "positions": -1, - "intermediate_tensors": 0, - "inputs_embeds": 0, - "hidden_states": 0, - } -) -class Qwen4ExpMultiTokenPredictor(nn.Module): - hf_to_vllm_mapper = Qwen3_5Model.hf_to_vllm_mapper | _EXTRA_WEIGHTS_MAPPER - - def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None: - super().__init__() - - model_config = vllm_config.model_config - config: Qwen4ExpTextConfig = model_config.hf_text_config - - self.config = config - # MoE SP also enables HC SP, while HC SP can be enabled independently. - self.use_hc_sequence_parallel = is_hc_sequence_parallel_enabled(vllm_config) - self.vocab_size = config.vocab_size - - self.mtp_start_layer_idx = config.num_hidden_layers - self.num_mtp_layers = getattr(config, "mtp_num_hidden_layers", 1) - - self.hidden_size = config.hidden_size - self.hc_count = config.hc_count - - self.embed_tokens = VocabParallelEmbedding(self.vocab_size, self.hidden_size) - draft_vllm_config = _make_draft_vllm_config( - vllm_config, - self.mtp_start_layer_idx, - ) - with set_current_vllm_config(draft_vllm_config, prefix=prefix): - # residual_linear_shared fusion: fc_embedding projects the token - # embedding, fc_hidden (shared across HC branches) projects the - # backbone hidden; the embedding is added as a residual to every - # branch (see mtp_residual_linear_shared.md). - self.fc_embedding = ColumnParallelLinear( - self.hidden_size, - self.hidden_size, - gather_output=True, - bias=False, - return_bias=False, - quant_config=draft_vllm_config.quant_config, - prefix=f"{prefix}.fc_embedding", - ) - self.fc_hidden = ColumnParallelLinear( - self.hidden_size, - self.hidden_size, - gather_output=True, - bias=False, - return_bias=False, - quant_config=draft_vllm_config.quant_config, - prefix=f"{prefix}.fc_hidden", - ) - self.layers = nn.ModuleList( - Qwen4ExpDecoderLayer( - draft_vllm_config, - layer_type="qwen_sparse_attention", - prefix=f"{prefix}.layers.{self.mtp_start_layer_idx + idx}", - ) - for idx in range(self.num_mtp_layers) - ) - self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible( - self.layers, - Qwen4ExpSparseMoeBlock, - "mlp", - ) - - self.pre_fc_norm_embedding = GemmaRMSNorm( - self.hidden_size, eps=config.rms_norm_eps - ) - self.pre_fc_norm_hidden = GemmaRMSNorm( - self.hidden_size * self.hc_count, eps=config.rms_norm_eps - ) - # HC final mixer collapses the multi stream into [T, H] for the LM head. - hc_config = HyperConnectionConfig( - hc_count=config.hc_count, - hidden_size=config.hidden_size, - params_dtype=torch.bfloat16, - hc_lowrank=config.hc_lowrank, - rms_norm_eps=config.rms_norm_eps, - hc_per_branch_norm=True, - ) - self.hyper_connection_mixer = GatedResidual( - hc_config, - use_combine=False, - prefix=maybe_prefix(prefix, "hyper_connection_mixer"), - ) - self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory( - ["hidden_states"], self.hidden_size * self.hc_count - ) - - def _iter_qsa_attentions(self): - """Yield MTP attention modules that own a QSA indexer.""" - for layer in self.layers: - attention = getattr(layer, "self_attn", None) - if ( - attention is not None - and getattr(attention, "indexer", None) is not None - ): - yield attention - - def set_skip_topk(self, skip: bool) -> None: - """Select on MTP step 0 and reuse its QSA indices on later steps.""" - for attention in self._iter_qsa_attentions(): - attention.indexer.skip_topk = skip - - def compact_topk_indices(self, row_indices: torch.Tensor) -> None: - """Keep each request's target-aligned step-0 sparse-index row.""" - num_rows = row_indices.numel() - for attention in self._iter_qsa_attentions(): - buffer = attention.topk_indices_buffer - selected = buffer.index_select(0, row_indices) - buffer[:num_rows].copy_(selected) - - def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: - return self.embed_tokens(input_ids) - - def forward( - self, - input_ids: torch.Tensor | None, - positions: torch.Tensor, - hidden_states: torch.Tensor | None = None, - intermediate_tensors: IntermediateTensors | None = None, - inputs_embeds: torch.Tensor | None = None, - spec_step_idx: int = 0, - ) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor] | IntermediateTensors: - hc_count = self.hc_count - hidden_size = self.hidden_size - - if get_pp_group().is_first_rank: - assert hidden_states is not None - if inputs_embeds is None: - assert input_ids is not None - inputs_embeds = self.embed_input_ids(input_ids) - # Embedding branch: pre-norm -> fc_embedding -> [T, H]. - inputs_embeds = self.pre_fc_norm_embedding(inputs_embeds) - inputs_embeds = self.fc_embedding(inputs_embeds) - - # Backbone hidden is multi-stream [T, hc_count*H] (scheme A: - # the main model truly emits the pre-final-mixer multi stream - # on the first step; subsequent steps reuse the prior draft - # step's multi stream). - num_tokens = hidden_states.shape[0] - hidden_states = hidden_states.view(num_tokens, hc_count, hidden_size) - hidden_states = self.pre_fc_norm_hidden(hidden_states.flatten(-2)).view( - num_tokens, hc_count, hidden_size - ) - hidden_states = self.fc_hidden(hidden_states) - # Add the embedding residual to every branch, then fold back - # to [T, hc_count*H] (HC outer, HS inner) for the HC decoder. - hidden_states = inputs_embeds.unsqueeze(-2) + hidden_states - hidden_states = hidden_states.flatten(-2) - else: - assert intermediate_tensors is not None - hidden_states = intermediate_tensors["hidden_states"] - - current_step_idx = spec_step_idx % self.num_mtp_layers - full_num_tokens = positions.shape[-1] - if self.use_hc_sequence_parallel: - hidden_states = sp_shard(hidden_states) - layer = self.layers[current_step_idx] - hidden_states, block_output, injection = layer( - hidden_states=hidden_states, - prev_block_output=None, - prev_injection=None, - positions=positions, - input_ids=None, - query_start_loc=None, - ngram_context=None, - ) - if not get_pp_group().is_last_rank: - # As in the target model, PP carries a materialized tensor rather - # than the delayed hidden/output/injection tuple. - hidden_states = layer.mlp_hyper_connection.combine( - hidden_states, block_output, injection - ) - return IntermediateTensors({"hidden_states": hidden_states}) - - # Last PP rank finalize. Keep both: - # (A) sample_hidden_states [T, H] -> single stream for the LM head - # (B) multi_hidden [T, hc_count*H] -> pre-final-mixer multi stream - # for the next draft step (zero extra compute, just kept). - multi_hidden, sample_hidden_states, _ = ( - self.hyper_connection_mixer.combine_and_mix( - hidden_states, block_output, injection - ) - ) - if self.use_hc_sequence_parallel: - # Gather both outputs together, then restore their contiguous layouts. - packed_hidden_states = torch.cat( - [sample_hidden_states, multi_hidden], dim=-1 - ) - packed_hidden_states = sp_all_gather(packed_hidden_states)[:full_num_tokens] - sample_hidden_states, multi_hidden = packed_hidden_states.split( - [hidden_size, hc_count * hidden_size], dim=-1 - ) - sample_hidden_states = sample_hidden_states.contiguous() - multi_hidden = multi_hidden.contiguous() - return sample_hidden_states, multi_hidden - - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - weights = maybe_fuse_shared_experts( - weights, - enabled=self.is_fused_shared_expert_enabled, - n_routed_experts=getattr(self.config, "num_experts", 0) or 0, - n_shared_experts=1, - ckpt_prefix="mlp.shared_expert", - ) - mapper = self.hf_to_vllm_mapper | WeightsMapper( - orig_to_new_substr={"hyper_connection_mixer.block_inject_weight": None} - ) - loader = AutoWeightsLoader( - self, - ignore_unexpected_suffixes=_QWEN4_EXP_IGNORED_MISSING_SUFFIXES.copy(), - ) - return loader.load_weights(weights, mapper=mapper) - - -@support_torch_compile( - dynamic_arg_dims={ - "input_ids": 0, - "positions": -1, - "intermediate_tensors": 0, - "inputs_embeds": 0, - "hidden_states": 0, - } -) -class Qwen4ExpMTP( - LocalArgmaxMixin, - nn.Module, - SupportsHCSP, - SupportsPP, - Qwen4ExpMixtureOfExperts, -): - packed_modules_mapping = { - "qkv_proj": ["q_proj", "k_proj", "v_proj"], - "gate_up_proj": ["gate_proj", "up_proj"], - "in_proj_qkvz": ["in_proj_qkv", "in_proj_z"], - "in_proj_ba": ["in_proj_b", "in_proj_a"], - "input_mix_weight_down_block_inject": [ - "input_mix_weight_down", - "block_inject_weight", - "_input_mix_padding", - ], - } - - def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None: - config: Qwen4ExpTextConfig = vllm_config.model_config.hf_text_config - self.vllm_config = vllm_config - self.quant_config = vllm_config.quant_config - - super().__init__() - self.config = config - self.model = Qwen4ExpMultiTokenPredictor( - vllm_config=vllm_config, - prefix=maybe_prefix(prefix, "mtp"), - ) - - if get_pp_group().is_last_rank: - if config.tie_word_embeddings: - self.lm_head = self.model.embed_tokens - else: - self.lm_head = ParallelLMHead( - config.vocab_size, - config.hidden_size, - prefix=maybe_prefix(prefix, "lm_head"), - ) - else: - self.lm_head = PPMissingLayer() - - self.logits_processor = LogitsProcessor(config.vocab_size) - self.make_empty_intermediate_tensors = ( - self.model.make_empty_intermediate_tensors - ) - self.set_moe_parameters(self.model.layers) - enable_qwen4_exp_low_latency_gemm(self, vllm_config.model_config.dtype) - - def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: - return self.model.embed_input_ids(input_ids) - - def forward( - self, - input_ids: torch.Tensor | None, - positions: torch.Tensor, - hidden_states: torch.Tensor | None = None, - intermediate_tensors: IntermediateTensors | None = None, - inputs_embeds: torch.Tensor | None = None, - spec_step_idx: int = 0, - ) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor] | IntermediateTensors: - return self.model( - input_ids, - positions, - hidden_states, - intermediate_tensors, - inputs_embeds, - spec_step_idx=spec_step_idx, - ) - - def compute_logits( - self, hidden_states: torch.Tensor, spec_step_idx: int = 0 - ) -> torch.Tensor | None: - return self.logits_processor(self.lm_head, hidden_states) - - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - def remap_weight_names(): - for name, weight in weights: - remapped_name = _remap_mtp_weight_name(name) - if remapped_name is not None: - yield remapped_name, weight - - mapper = WeightsMapper( - orig_to_new_substr={"hyper_connection_mixer.block_inject_weight": None} - ) - loader = AutoWeightsLoader( - self, - ignore_unexpected_suffixes=_QWEN4_EXP_IGNORED_MISSING_SUFFIXES.copy(), - ) - return loader.load_weights(remap_weight_names(), mapper=mapper) - - -__all__ = ["Qwen4ExpMTP", "Qwen4ExpMultiTokenPredictor"] diff --git a/vllm/models/qwen4_exp/amd/ops/__init__.py b/vllm/models/qwen4_exp/amd/ops/__init__.py deleted file mode 100644 index 56c33274ef..0000000000 --- a/vllm/models/qwen4_exp/amd/ops/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""AMD ROCm Qwen4Exp kernels; import leaf modules directly.""" diff --git a/vllm/models/qwen4_exp/amd/ops/hc.py b/vllm/models/qwen4_exp/amd/ops/hc.py deleted file mode 100644 index 1faf89b020..0000000000 --- a/vllm/models/qwen4_exp/amd/ops/hc.py +++ /dev/null @@ -1,487 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""AMD ROCm HyperConnection kernels for Qwen4Exp.""" - -import torch - -from vllm.platforms import current_platform -from vllm.triton_utils import tl, triton -from vllm.utils.torch_utils import direct_register_custom_op - - -@triton.jit -def _grouped_gemma_rmsnorm_kernel( - x_ptr, - w_ptr, - y_ptr, - stride_x, - stride_y, - DIM: tl.constexpr, - NUM_GROUPS: tl.constexpr, - W_SHARED: tl.constexpr, - EPS: tl.constexpr, - launch_pdl: tl.constexpr, -) -> None: - GROUP_DIM: tl.constexpr = DIM // NUM_GROUPS - BLOCK_SIZE: tl.constexpr = triton.next_power_of_2(GROUP_DIM) - - pid = tl.program_id(0) - group_id = pid % NUM_GROUPS - row = pid // NUM_GROUPS - - offs_g = tl.arange(0, BLOCK_SIZE) - offsets = group_id * GROUP_DIM + offs_g - mask = offs_g < GROUP_DIM - # A [GROUP_DIM] affine is shared; a [DIM] affine follows the grouped - # checkpoint layout. - w_offs = offs_g if W_SHARED else offsets - - if launch_pdl: - tl.extra.cuda.gdc_wait() - - x = tl.load(x_ptr + row * stride_x + offsets, mask, other=0.0).to(tl.float32) - w = tl.load(w_ptr + w_offs, mask, other=0.0) - - rrms = tl.rsqrt(tl.sum(x * x) / GROUP_DIM + EPS) - # Gemma's (1 + w) affine is written this way to lower to an FMA. - y = x * rrms - y += y * w.to(tl.float32) - - if launch_pdl: - tl.extra.cuda.gdc_launch_dependents() - tl.store(y_ptr + row * stride_y + offsets, y, mask) - - -def _grouped_gemma_rmsnorm( - x: torch.Tensor, weight: torch.Tensor, eps: float, num_groups: int -) -> torch.Tensor: - N, DIM = x.shape - assert x.stride(1) == 1, "grouped Gemma RMSNorm requires unit inner stride" - assert weight.is_contiguous(), "grouped Gemma RMSNorm weight must be contiguous" - assert DIM % num_groups == 0 - group_dim = DIM // num_groups - assert weight.numel() in (group_dim, DIM) - - y = x.new_empty(x.shape) - _grouped_gemma_rmsnorm_kernel[(N * num_groups,)]( - x, - weight, - y, - x.stride(0), - y.stride(0), - DIM, - num_groups, - W_SHARED=weight.numel() == group_dim, - EPS=eps, - launch_pdl=current_platform.is_arch_support_pdl(), - ) - return y - - -@triton.jit -def _hc_silu_kernel( - x_ptr, - y_ptr, - stride_x, - stride_y, - DIM: tl.constexpr, - HC: tl.constexpr, - launch_pdl: tl.constexpr, -) -> None: - BLOCK_SIZE: tl.constexpr = triton.next_power_of_2(DIM) - - row = tl.program_id(0) - offs = tl.arange(0, BLOCK_SIZE) - mask = offs < DIM - - if launch_pdl: - tl.extra.cuda.gdc_wait() - - x = tl.load(x_ptr + row * stride_x + offs, mask).to(tl.float32) / HC - y = x * tl.sigmoid(x) - - if launch_pdl: - tl.extra.cuda.gdc_launch_dependents() - tl.store(y_ptr + row * stride_y + offs, y, mask) - - -def _hc_silu(x: torch.Tensor, hc_count: int) -> torch.Tensor: - num_tokens, DIM = x.shape - assert x.stride(1) == 1 - - output = x.new_empty(x.shape) - _hc_silu_kernel[(num_tokens,)]( - x, - output, - x.stride(0), - output.stride(0), - DIM=DIM, - HC=hc_count, - launch_pdl=current_platform.is_arch_support_pdl(), - ) - return output - - -@triton.jit -def _hc_gate_mix_kernel( - x_ptr, - g_ptr, - y_ptr, - stride_x, - stride_g, - stride_y, - DIM: tl.constexpr, - HC: tl.constexpr, - BLOCK_SIZE: tl.constexpr, - launch_pdl: tl.constexpr, -) -> None: - HC_DIM: tl.constexpr = DIM // HC - - row = tl.program_id(0) - tile_id = tl.program_id(1) - offs_inner = tile_id * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE) - mask = offs_inner < HC_DIM - - if launch_pdl: - tl.extra.cuda.gdc_wait() - - # The constexpr loop is unrolled and keeps one stream live at a time. - # Materializing [HC, BLOCK_SIZE] more than doubles latency at large M. - acc = tl.zeros([BLOCK_SIZE], dtype=tl.float32) - for stream in tl.static_range(HC): - offsets = stream * HC_DIM + offs_inner - g = tl.load(g_ptr + row * stride_g + offsets, mask, other=0.0) - x = tl.load(x_ptr + row * stride_x + offsets, mask, other=0.0) - acc += tl.sigmoid(g.to(tl.float32)) * x.to(tl.float32) - acc /= HC - - if launch_pdl: - tl.extra.cuda.gdc_launch_dependents() - tl.store(y_ptr + row * stride_y + offs_inner, acc, mask) - - -def _hc_gate_mix(x: torch.Tensor, gate: torch.Tensor, hc_count: int) -> torch.Tensor: - N, DIM = gate.shape - assert x.shape == gate.shape - assert DIM % hc_count == 0 - assert x.stride(1) == 1 - assert gate.stride(1) == 1 - - HC_DIM = DIM // hc_count - out = x.new_empty(N, HC_DIM) - BLOCK_SIZE = 512 - _hc_gate_mix_kernel[(N, triton.cdiv(HC_DIM, BLOCK_SIZE))]( - x, - gate, - out, - x.stride(0), - gate.stride(0), - out.stride(0), - DIM, - hc_count, - BLOCK_SIZE, - launch_pdl=current_platform.is_arch_support_pdl(), - ) - return out - - -@triton.jit -def _hc_combine_kernel( - block_ptr, - res_ptr, - inj_ptr, - out_ptr, - stride_block, - stride_res, - stride_inj, - stride_out, - HC_DIM: tl.constexpr, - HC: tl.constexpr, - BLOCK_SIZE: tl.constexpr, - launch_pdl: tl.constexpr, -) -> None: - HC_PAD: tl.constexpr = triton.next_power_of_2(HC) - - row = tl.program_id(0) - tile_id = tl.program_id(1) - - offs_inner = tile_id * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE) - mask_inner = offs_inner < HC_DIM - offs_hc = tl.arange(0, HC_PAD) - mask_hc = offs_hc < HC - offs = offs_hc[:, None] * HC_DIM + offs_inner[None, :] - mask = mask_hc[:, None] & mask_inner[None, :] - - if launch_pdl: - tl.extra.cuda.gdc_wait() - - inj = tl.load(inj_ptr + row * stride_inj + offs_hc, mask_hc, other=0.0) - block = tl.load(block_ptr + row * stride_block + offs_inner, mask_inner, other=0.0) - res = tl.load(res_ptr + row * stride_res + offs, mask, other=0.0) - - # Keeping HC as a broadcast dimension is faster here than four separate - # residual load/store sequences. - inj = 2.0 * tl.sigmoid(inj.to(tl.float32) / HC) - out = res.to(tl.float32) + block.to(tl.float32)[None, :] * inj[:, None] - - if launch_pdl: - tl.extra.cuda.gdc_launch_dependents() - tl.store(out_ptr + row * stride_out + offs, out, mask=mask) - - -def _hc_combine( - residual: torch.Tensor, - block_output: torch.Tensor, - injection_logits: torch.Tensor, - hc_count: int, -) -> torch.Tensor: - N, DIM = residual.shape - assert DIM % hc_count == 0 - hc_dim = DIM // hc_count - assert block_output.shape == (N, hc_dim) - assert injection_logits.shape == (N, hc_count) - assert residual.stride(1) == 1 - assert block_output.stride(1) == 1 - assert injection_logits.stride(1) == 1 - - out = residual.new_empty(residual.shape) - BLOCK_SIZE = 512 - _hc_combine_kernel[(N, triton.cdiv(hc_dim, BLOCK_SIZE))]( - block_output, - residual, - injection_logits, - out, - block_output.stride(0), - residual.stride(0), - injection_logits.stride(0), - out.stride(0), - hc_dim, - hc_count, - BLOCK_SIZE, - launch_pdl=current_platform.is_arch_support_pdl(), - ) - return out - - -@triton.jit -def _hc_combine_norm_kernel( - block_ptr, - res_ptr, - inj_ptr, - w_ptr, - out_ptr, - y_ptr, - stride_block, - stride_res, - stride_inj, - stride_out, - stride_y, - HC_DIM: tl.constexpr, - HC: tl.constexpr, - W_SHARED: tl.constexpr, - EPS: tl.constexpr, - BLOCK_SIZE: tl.constexpr, - launch_pdl: tl.constexpr, -) -> None: - HC_PAD: tl.constexpr = triton.next_power_of_2(HC) - NUM_TILES: tl.constexpr = triton.cdiv(HC_DIM, BLOCK_SIZE) - NUM_TILES_PAD: tl.constexpr = triton.next_power_of_2(NUM_TILES) - - row = tl.program_id(0) - stream = tl.program_id(1) - offs_hc = tl.arange(0, HC_PAD) - mask_hc = offs_hc < HC - tile_ids = tl.arange(0, NUM_TILES_PAD) - offs_inner = tile_ids[:, None] * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)[None, :] - mask_inner = offs_inner < HC_DIM - offs = stream * HC_DIM + offs_inner - # Shared norm weights repeat across streams; per-branch weights use the - # same flattened HC layout as the residual. - w_offs = offs_inner if W_SHARED else offs - - if launch_pdl: - tl.extra.cuda.gdc_wait() - - # Start the uncached residual load first, then issue the other combine - # loads before consuming any of them. - res = tl.load(res_ptr + row * stride_res + offs, mask_inner, other=0.0) - inj = tl.load(inj_ptr + row * stride_inj + offs_hc, mask_hc, other=0.0) - block = tl.load(block_ptr + row * stride_block + offs_inner, mask_inner, other=0.0) - inj = 2.0 * tl.sigmoid(inj.to(tl.float32) / HC) - inj = tl.sum(tl.where(offs_hc == stream, inj, 0.0)) - # Round the materialized combine result before normalization. This matches - # the unfused combine -> RMSNorm boundary. - out = (res.to(tl.float32) + block.to(tl.float32) * inj).to(out_ptr.dtype.element_ty) - tl.store(out_ptr + row * stride_out + offs, out, mask=mask_inner) - - out = out.to(tl.float32) - # Keep the two-axis reduction: flattening the padded tile is ~40% slower - # at decode sizes. - sum_sq = tl.sum(tl.sum(out * out, axis=1), axis=0) - rrms = tl.rsqrt(sum_sq / HC_DIM + EPS) - - if launch_pdl: - tl.extra.cuda.gdc_launch_dependents() - - # Loading the weight earlier helps decode but keeps the tile live across - # the reduction and regresses larger batches, so defer it to the norm. - w = tl.load(w_ptr + w_offs, mask_inner, other=0.0) - y = out * rrms - y += y * w.to(tl.float32) - tl.store(y_ptr + row * stride_y + offs, y, mask_inner) - - -def _hc_combine_norm( - residual: torch.Tensor, - block_output: torch.Tensor, - injection_logits: torch.Tensor, - norm_weight: torch.Tensor, - eps: float, - hc_count: int, -) -> tuple[torch.Tensor, torch.Tensor]: - N, DIM = residual.shape - assert DIM % hc_count == 0 - hc_dim = DIM // hc_count - assert block_output.shape == (N, hc_dim) - assert injection_logits.shape == (N, hc_count) - assert residual.stride(1) == 1 - assert block_output.stride(1) == 1 - assert injection_logits.stride(1) == 1 - assert norm_weight.is_contiguous() - assert norm_weight.numel() in (hc_dim, DIM) - - out = residual.new_empty(residual.shape) - y = residual.new_empty(residual.shape) - BLOCK_SIZE = 512 - _hc_combine_norm_kernel[(N, hc_count)]( - block_output, - residual, - injection_logits, - norm_weight, - out, - y, - block_output.stride(0), - residual.stride(0), - injection_logits.stride(0), - out.stride(0), - y.stride(0), - hc_dim, - hc_count, - W_SHARED=norm_weight.numel() == hc_dim, - EPS=eps, - BLOCK_SIZE=BLOCK_SIZE, - launch_pdl=current_platform.is_arch_support_pdl(), - ) - return out, y - - -def _same_shape_fake(x: torch.Tensor, *args) -> torch.Tensor: - return x.new_empty(x.shape) - - -def _hc_gate_mix_fake( - x: torch.Tensor, gate: torch.Tensor, hc_count: int -) -> torch.Tensor: - del gate - return x.new_empty((x.shape[0], x.shape[1] // hc_count)) - - -def _hc_combine_fake( - residual: torch.Tensor, - block_output: torch.Tensor, - injection_logits: torch.Tensor, - hc_count: int, -) -> torch.Tensor: - del block_output, injection_logits, hc_count - return residual.new_empty(residual.shape) - - -def _hc_combine_norm_fake( - residual: torch.Tensor, - block_output: torch.Tensor, - injection_logits: torch.Tensor, - norm_weight: torch.Tensor, - eps: float, - hc_count: int, -) -> tuple[torch.Tensor, torch.Tensor]: - del block_output, injection_logits, norm_weight, eps, hc_count - return residual.new_empty(residual.shape), residual.new_empty(residual.shape) - - -direct_register_custom_op( - op_name="qwen4_exp_grouped_gemma_rmsnorm", - op_func=_grouped_gemma_rmsnorm, - fake_impl=_same_shape_fake, -) -direct_register_custom_op( - op_name="qwen4_exp_hc_silu", - op_func=_hc_silu, - fake_impl=_same_shape_fake, -) -direct_register_custom_op( - op_name="qwen4_exp_hc_gate_mix", - op_func=_hc_gate_mix, - fake_impl=_hc_gate_mix_fake, -) -direct_register_custom_op( - op_name="qwen4_exp_hc_combine", - op_func=_hc_combine, - fake_impl=_hc_combine_fake, -) -direct_register_custom_op( - op_name="qwen4_exp_hc_combine_norm", - op_func=_hc_combine_norm, - fake_impl=_hc_combine_norm_fake, -) - - -def grouped_gemma_rmsnorm( - x: torch.Tensor, weight: torch.Tensor, eps: float, num_groups: int -) -> torch.Tensor: - return torch.ops.vllm.qwen4_exp_grouped_gemma_rmsnorm(x, weight, eps, num_groups) - - -def hc_silu(x: torch.Tensor, hc_count: int) -> torch.Tensor: - return torch.ops.vllm.qwen4_exp_hc_silu(x, hc_count) - - -def hc_gate_mix(x: torch.Tensor, gate: torch.Tensor, hc_count: int) -> torch.Tensor: - return torch.ops.vllm.qwen4_exp_hc_gate_mix(x, gate, hc_count) - - -def hc_combine( - residual: torch.Tensor, - block_output: torch.Tensor, - injection_logits: torch.Tensor, - hc_count: int, -) -> torch.Tensor: - return torch.ops.vllm.qwen4_exp_hc_combine( - residual, block_output, injection_logits, hc_count - ) - - -def hc_combine_norm( - residual: torch.Tensor, - block_output: torch.Tensor, - injection_logits: torch.Tensor, - norm_weight: torch.Tensor, - eps: float, - hc_count: int, -) -> tuple[torch.Tensor, torch.Tensor]: - return torch.ops.vllm.qwen4_exp_hc_combine_norm( - residual, - block_output, - injection_logits, - norm_weight, - eps, - hc_count, - ) - - -__all__ = [ - "grouped_gemma_rmsnorm", - "hc_combine", - "hc_combine_norm", - "hc_gate_mix", - "hc_silu", -] diff --git a/vllm/models/qwen4_exp/amd/ops/qsa.py b/vllm/models/qwen4_exp/amd/ops/qsa.py deleted file mode 100644 index 6bdb1fe592..0000000000 --- a/vllm/models/qwen4_exp/amd/ops/qsa.py +++ /dev/null @@ -1,1122 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Triton kernels for the Qwen4Exp weight-free QSA path.""" - -from __future__ import annotations - -import math - -import torch - -from vllm import _custom_ops as ops -from vllm.platforms import current_platform -from vllm.triton_utils import HAS_TRITON, tl, triton - -_LOGITS_WORKSPACE_BYTES = 128 * 1024 * 1024 -_TOPK_WORKSPACE_BYTES = 1024 * 1024 - - -@triton.jit -def _qsa_mqa_paged_kernel( - q_ptr, - k_cache_ptr, - page_table_ptr, - token_to_req_ptr, - query_positions_ptr, - sequence_lengths_ptr, - visible_blocks_ptr, - logits_ptr, - stride_q_row, - stride_q_head, - stride_q_dim, - stride_cache_block, - stride_cache_token, - stride_cache_dim, - stride_table_req, - stride_table_page, - stride_logits_row, - num_rows, - num_columns, - num_pages, - num_requests, - score_divisor, - PAGE_SIZE: tl.constexpr, - PAGE_TABLE_WIDTH: tl.constexpr, - NUM_HEADS: tl.constexpr, - HEAD_DIM: tl.constexpr, - BLOCK_N: tl.constexpr, - BLOCK_D: tl.constexpr, - COMPRESS_RATIO: tl.constexpr, -) -> None: - row = tl.program_id(0) - columns = tl.program_id(1) * BLOCK_N + tl.arange(0, BLOCK_N) - dims = tl.arange(0, BLOCK_D) - request = tl.load(token_to_req_ptr + row) - safe_request = tl.minimum(tl.maximum(request, 0), num_requests - 1) - query_position = tl.load(query_positions_ptr + row) - sequence_length = tl.load( - sequence_lengths_ptr + safe_request, - mask=(request >= 0) & (request < num_requests), - other=0, - ) - visible = tl.minimum( - (query_position + 1) // COMPRESS_RATIO, - sequence_length // COMPRESS_RATIO, - ) - if tl.program_id(1) == 0: - tl.store(visible_blocks_ptr + row, visible) - logical_page = columns // PAGE_SIZE - page_offset = columns % PAGE_SIZE - valid = ( - (row < num_rows) - & (columns < num_columns) - & (columns < visible) - & (request >= 0) - & (request < num_requests) - & (logical_page < PAGE_TABLE_WIDTH) - ) - safe_logical_page = tl.minimum(logical_page, PAGE_TABLE_WIDTH - 1) - physical_page = tl.load( - page_table_ptr - + safe_request * stride_table_req - + safe_logical_page * stride_table_page, - mask=valid, - other=-1, - ) - valid &= (physical_page >= 0) & (physical_page < num_pages) - # physical_page * block stride can overflow int32 for large caches. - safe_physical_page = tl.maximum(physical_page, 0).to(tl.int64) - score = tl.zeros((BLOCK_N,), dtype=tl.float32) - - for head in tl.static_range(0, NUM_HEADS): - query = tl.load( - q_ptr + row * stride_q_row + head * stride_q_head + dims * stride_q_dim, - mask=dims < HEAD_DIM, - other=0.0, - ).to(tl.float32) - keys = tl.load( - k_cache_ptr - + safe_physical_page[:, None] * stride_cache_block - + page_offset[:, None] * stride_cache_token - + dims[None, :] * stride_cache_dim, - mask=valid[:, None] & (dims[None, :] < HEAD_DIM), - other=0.0, - ).to(tl.float32) - dot = tl.sum(keys * query[None, :], axis=1) - score += tl.maximum(dot, 0.0) - - score /= score_divisor - tl.store( - logits_ptr + row * stride_logits_row + columns, - tl.where(valid, score, -float("inf")), - mask=(row < num_rows) & (columns < num_columns), - ) - - -@triton.jit -def _expand_qsa_indices_kernel( - block_indices_ptr, - query_positions_ptr, - sequence_lengths_ptr, - token_to_req_ptr, - output_ptr, - stride_blocks_row, - stride_blocks_column, - stride_output_row, - stride_output_column, - rows, - num_requests, - BLOCK_TOPK: tl.constexpr, - COMPRESS_RATIO: tl.constexpr, - TOKEN_TOPK: tl.constexpr, - OUTPUT_WIDTH: tl.constexpr, - COLUMN_BLOCK: tl.constexpr, -) -> None: - row = tl.program_id(0) - columns = tl.program_id(1) * COLUMN_BLOCK + tl.arange(0, COLUMN_BLOCK) - query_position = tl.load(query_positions_ptr + row) - request = tl.load(token_to_req_ptr + row) - safe_request = tl.minimum(tl.maximum(request, 0), num_requests - 1) - sequence_length = tl.load( - sequence_lengths_ptr + safe_request, - mask=(request >= 0) & (request < num_requests), - other=0, - ) - complete_blocks = tl.minimum( - tl.minimum( - (query_position + 1) // COMPRESS_RATIO, - sequence_length // COMPRESS_RATIO, - ), - BLOCK_TOPK, - ) - expanded_count = complete_blocks * COMPRESS_RATIO - tail_start = ((query_position + 1) // COMPRESS_RATIO) * COMPRESS_RATIO - tail_count = (query_position + 1) - tail_start - - is_expanded = columns < expanded_count - block_rank = columns // COMPRESS_RATIO - offset = columns % COMPRESS_RATIO - safe_rank = tl.minimum(block_rank, BLOCK_TOPK - 1) - block = tl.load( - block_indices_ptr + row * stride_blocks_row + safe_rank * stride_blocks_column, - mask=(row < rows) & is_expanded, - other=-1, - ) - expanded = block * COMPRESS_RATIO + offset - tail_offset = columns - expanded_count - is_tail = ( - (columns >= expanded_count) - & (tail_offset < tail_count) - & (tail_offset < COMPRESS_RATIO - 1) - ) - token = tl.where(is_expanded, expanded, tail_start + tail_offset) - valid = ( - (row < rows) - & (columns < OUTPUT_WIDTH) - & (is_expanded | is_tail) - & (token >= 0) - & (token < sequence_length) - ) - tl.store( - output_ptr + row * stride_output_row + columns * stride_output_column, - tl.where(valid, token, -1), - mask=(row < rows) & (columns < OUTPUT_WIDTH), - ) - - -@triton.jit -def _qsa_sparse_paged_gqa_splitk_kernel( - q_ptr, - k_cache_ptr, - v_cache_ptr, - indices_ptr, - block_table_ptr, - token_to_req_ptr, - partial_output_ptr, - partial_lse_ptr, - output_ptr, - stride_q_row, - stride_q_head, - stride_k_block, - stride_k_token, - stride_k_head, - stride_v_block, - stride_v_token, - stride_v_head, - stride_indices_row, - stride_table_req, - stride_output_row, - stride_output_head, - num_rows, - num_cache_blocks, - num_requests, - TOPK: tl.constexpr, - PAGE_SIZE: tl.constexpr, - PAGE_TABLE_WIDTH: tl.constexpr, - GROUP_SIZE: tl.constexpr, - HEAD_DIM: tl.constexpr, - NUM_QUERY_HEADS: tl.constexpr, - NUM_SPLITS: tl.constexpr, - NUM_TILES: tl.constexpr, - BLOCK_M: tl.constexpr, - BLOCK_N: tl.constexpr, -) -> None: - row = tl.program_id(0) - kv_head = tl.program_id(1) - split_id = tl.program_id(2) - request = tl.load(token_to_req_ptr + row) - safe_request = tl.minimum(tl.maximum(request, 0), num_requests - 1) - - head_offsets = tl.arange(0, BLOCK_M) - dim_offsets = tl.arange(0, HEAD_DIM) - column_offsets = tl.arange(0, BLOCK_N) - first_head = kv_head * GROUP_SIZE - query = tl.load( - q_ptr - + row * stride_q_row - + (first_head + head_offsets[:, None]) * stride_q_head - + dim_offsets[None, :], - mask=head_offsets[:, None] < GROUP_SIZE, - other=0.0, - ) - - max_value = tl.full((BLOCK_M,), -1.0e20, dtype=tl.float32) - normalizer = tl.zeros((BLOCK_M,), dtype=tl.float32) - accumulator = tl.zeros((BLOCK_M, HEAD_DIM), dtype=tl.float32) - softmax_scale_log2: tl.constexpr = (HEAD_DIM**-0.5) * 1.4426950408889634 - - # Dynamic bounds avoid padded main-loop iterations for uneven splits. - split_tile_start = split_id * NUM_TILES // NUM_SPLITS - split_tile_end = (split_id + 1) * NUM_TILES // NUM_SPLITS - for tile in range(split_tile_start, split_tile_end): - columns = tile * BLOCK_N + column_offsets - logical_token = tl.load( - indices_ptr + row * stride_indices_row + columns, - mask=columns < TOPK, - other=-1, - ) - safe_token = tl.maximum(logical_token, 0) - logical_page = safe_token // PAGE_SIZE - page_offset = safe_token % PAGE_SIZE - valid = ( - (request >= 0) - & (request < num_requests) - & (logical_token >= 0) - & (logical_page < PAGE_TABLE_WIDTH) - ) - physical_page = tl.load( - block_table_ptr - + safe_request * stride_table_req - + tl.minimum(logical_page, PAGE_TABLE_WIDTH - 1), - mask=valid, - other=-1, - ) - valid &= (physical_page >= 0) & (physical_page < num_cache_blocks) - # physical_page * block stride can overflow int32 for large caches. - safe_page = tl.maximum(physical_page, 0).to(tl.int64) - keys = tl.load( - k_cache_ptr - + safe_page[None, :] * stride_k_block - + page_offset[None, :] * stride_k_token - + kv_head * stride_k_head - + dim_offsets[:, None], - mask=valid[None, :], - other=0.0, - ) - values = tl.load( - v_cache_ptr - + safe_page[:, None] * stride_v_block - + page_offset[:, None] * stride_v_token - + kv_head * stride_v_head - + dim_offsets[None, :], - mask=valid[:, None], - other=0.0, - ) - scores = tl.dot(query, keys) - # Scaling scores avoids re-quantizing a scaled query to BF16. - scores *= softmax_scale_log2 - scores = tl.where(valid[None, :], scores, -1.0e20) - next_max = tl.maximum(max_value, tl.max(scores, axis=1)) - alpha = tl.math.exp2(max_value - next_max) - probabilities = tl.where( - valid[None, :], tl.math.exp2(scores - next_max[:, None]), 0.0 - ) - accumulator = tl.dot( - probabilities.to(values.dtype), - values, - acc=accumulator * alpha[:, None], - ) - normalizer = normalizer * alpha + tl.sum(probabilities, axis=1) - max_value = next_max - - has_values = normalizer > 0 - normalized_output = tl.where( - has_values[:, None], - accumulator / tl.maximum(normalizer[:, None], 1.0e-20), - 0.0, - ) - output_mask = head_offsets[:, None] < GROUP_SIZE - if NUM_SPLITS == 1: - tl.store( - output_ptr - + row * stride_output_row - + (first_head + head_offsets[:, None]) * stride_output_head - + dim_offsets[None, :], - normalized_output, - mask=output_mask, - ) - else: - partial_lse = tl.where( - has_values, - max_value + tl.math.log2(tl.maximum(normalizer, 1.0e-20)), - -float("inf"), - ) - tl.store( - partial_output_ptr - + ( - (split_id * num_rows + row) * NUM_QUERY_HEADS - + first_head - + head_offsets[:, None] - ) - * HEAD_DIM - + dim_offsets[None, :], - normalized_output, - mask=output_mask, - ) - tl.store( - partial_lse_ptr - + (split_id * num_rows + row) * NUM_QUERY_HEADS - + first_head - + head_offsets, - partial_lse, - mask=head_offsets < GROUP_SIZE, - ) - - -@triton.jit -def _qsa_merge_splitk_kernel( - partial_output_ptr, - partial_lse_ptr, - output_ptr, - stride_output_row, - stride_output_head, - num_rows, - HEAD_DIM: tl.constexpr, - NUM_QUERY_HEADS: tl.constexpr, - NUM_SPLITS: tl.constexpr, - BLOCK_SPLITS: tl.constexpr, -) -> None: - row = tl.program_id(0) - head = tl.program_id(1) - split_offsets = tl.arange(0, BLOCK_SPLITS) - dim_offsets = tl.arange(0, HEAD_DIM) - split_mask = split_offsets < NUM_SPLITS - lse = tl.load( - partial_lse_ptr + (split_offsets * num_rows + row) * NUM_QUERY_HEADS + head, - mask=split_mask, - other=-float("inf"), - ) - lse_max = tl.max(lse, axis=0) - has_values = lse_max > -float("inf") - shifted = tl.where(split_mask & has_values, lse - lse_max, -float("inf")) - weights = tl.math.exp2(shifted) - denominator = tl.sum(weights, axis=0) - partial_output = tl.load( - partial_output_ptr - + ((split_offsets[:, None] * num_rows + row) * NUM_QUERY_HEADS + head) - * HEAD_DIM - + dim_offsets[None, :], - mask=split_mask[:, None], - other=0.0, - ) - merged = tl.sum(partial_output * weights[:, None], axis=0) - merged = tl.where(denominator > 0, merged / denominator, 0.0) - tl.store( - output_ptr + row * stride_output_row + head * stride_output_head + dim_offsets, - merged, - ) - - -@triton.jit -def _store_qsa_rows_kernel( - cache_ptr, - slots_ptr, - rows_ptr, - stride_cache_block, - stride_cache_token, - stride_cache_dim, - stride_rows_row, - stride_rows_dim, - num_rows, - num_blocks, - PAGE_SIZE: tl.constexpr, - WIDTH: tl.constexpr, - BLOCK_D: tl.constexpr, -) -> None: - row = tl.program_id(0) - dims = tl.arange(0, BLOCK_D) - slot = tl.load(slots_ptr + row) - valid = (row < num_rows) & (slot >= 0) & (slot < num_blocks * PAGE_SIZE) - block = tl.maximum(slot, 0) // PAGE_SIZE - token = tl.maximum(slot, 0) % PAGE_SIZE - values = tl.load( - rows_ptr + row * stride_rows_row + dims * stride_rows_dim, - mask=valid & (dims < WIDTH), - other=0, - ) - tl.store( - cache_ptr - + block * stride_cache_block - + token * stride_cache_token - + dims * stride_cache_dim, - values, - mask=valid & (dims < WIDTH), - ) - - -@triton.jit -def _compress_qsa_groups_kernel( - raw_keys_ptr, # this step's raw key rows, straight from activations - raw_positions_ptr, # this step's per-token positions - compressor_state_cache_ptr, # per-request ring of previous raw keys - rope_cache_ptr, # packed RoPE position tail of the ring - compressor_state_table_ptr, - token_to_req_ptr, - query_start_loc_ptr, - logical_positions_ptr, - compressed_slots_ptr, - pooled_ptr, - first_positions_ptr, - stride_raw_row, - stride_raw_dim, - stride_raw_positions_row, - stride_raw_positions_dim, - stride_compressor_state_block, - stride_compressor_state_token, - stride_compressor_state_dim, - stride_rope_block, - stride_rope_token, - stride_rope_dim, - stride_compressor_state_table_req, - stride_pooled_row, - stride_pooled_dim, - stride_positions_row, - stride_positions_dim, - num_rows, - num_compressor_state_blocks, - num_requests, - COMPRESSOR_STATE_SIZE: tl.constexpr, - COMPRESS_RATIO: tl.constexpr, - HEAD_DIM: tl.constexpr, - BLOCK_D: tl.constexpr, - LOAD_ROPE_POSITIONS: tl.constexpr, -) -> None: - row = tl.program_id(0) - dims = tl.arange(0, BLOCK_D) - request = tl.load(token_to_req_ptr + row) - end_position = tl.load(logical_positions_ptr + row) - compressed_slot = tl.load(compressed_slots_ptr + row) - valid_request = (request >= 0) & (request < num_requests) - safe_request = tl.minimum(tl.maximum(request, 0), num_requests - 1) - query_row_start = tl.load( - query_start_loc_ptr + safe_request, mask=valid_request, other=0 - ) - query_row_end = tl.load( - query_start_loc_ptr + safe_request + 1, mask=valid_request, other=0 - ) - chunk_start_position = end_position - (row - query_row_start) - compressor_state_block = tl.load( - compressor_state_table_ptr + safe_request * stride_compressor_state_table_req, - mask=valid_request, - other=-1, - ) - valid_compressor_state_block = (compressor_state_block >= 0) & ( - compressor_state_block < num_compressor_state_blocks - ) - valid_row = ( - (row < num_rows) - & valid_request - & (row >= query_row_start) - & (row < query_row_end) - & (end_position >= COMPRESS_RATIO - 1) - & (compressed_slot >= 0) - ) - accumulator = tl.zeros((BLOCK_D,), dtype=tl.float32) - - # A group can span the compressor-state ring (older members) and this - # step's raw rows (members at positions >= chunk_start_position). - for group_offset in tl.range(0, COMPRESS_RATIO): - position = end_position - (COMPRESS_RATIO - 1 - group_offset) - use_raw = position >= chunk_start_position - raw_row = query_row_start + position - chunk_start_position - raw_values = tl.load( - raw_keys_ptr + raw_row * stride_raw_row + dims * stride_raw_dim, - mask=valid_row - & use_raw - & (raw_row >= query_row_start) - & (raw_row < query_row_end) - & (raw_row < num_rows) - & (dims < HEAD_DIM), - other=0.0, - ).to(tl.float32) - compressor_state_values = tl.load( - compressor_state_cache_ptr - + tl.maximum(compressor_state_block, 0).to(tl.int64) - * stride_compressor_state_block - + (position % COMPRESSOR_STATE_SIZE) * stride_compressor_state_token - + dims * stride_compressor_state_dim, - mask=valid_row - & ~use_raw - & valid_compressor_state_block - & (dims < HEAD_DIM), - other=0.0, - ).to(tl.float32) - accumulator += tl.where(use_raw, raw_values, compressor_state_values) - - tl.store( - pooled_ptr + row * stride_pooled_row + dims * stride_pooled_dim, - accumulator / COMPRESS_RATIO, - mask=(row < num_rows) & (dims < HEAD_DIM), - ) - - position_dims = tl.arange(0, 4) - first_position = end_position - COMPRESS_RATIO + 1 - if LOAD_ROPE_POSITIONS: - first_from_raw = first_position >= chunk_start_position - raw_first_row = query_row_start + first_position - chunk_start_position - raw_position_values = tl.load( - raw_positions_ptr - + raw_first_row * stride_raw_positions_row - + position_dims * stride_raw_positions_dim, - mask=valid_row - & first_from_raw - & (raw_first_row >= query_row_start) - & (raw_first_row < query_row_end) - & (raw_first_row < num_rows) - & (position_dims < 3), - other=0, - ) - compressor_state_position_values = tl.load( - rope_cache_ptr - + tl.maximum(compressor_state_block, 0).to(tl.int64) * stride_rope_block - + (first_position % COMPRESSOR_STATE_SIZE) * stride_rope_token - + position_dims * stride_rope_dim, - mask=valid_row - & ~first_from_raw - & valid_compressor_state_block - & (position_dims < 3), - other=0, - ) - position_values = tl.where( - first_from_raw, - raw_position_values, - compressor_state_position_values, - ) - else: - position_values = tl.where(valid_row, first_position, 0) - tl.store( - first_positions_ptr - + row * stride_positions_row - + position_dims * stride_positions_dim, - position_values, - mask=(row < num_rows) & (position_dims < 3), - ) - - -def _validate_mqa(q: torch.Tensor) -> None: - if q.ndim != 3 or q.shape[1] <= 0 or q.shape[2] <= 0: - raise ValueError("QSA query must be [rows, heads, head_dim]") - - -def qsa_mqa_paged( - q: torch.Tensor, - k_cache: torch.Tensor, - page_table: torch.Tensor, - token_to_req: torch.Tensor, - query_positions: torch.Tensor, - sequence_lengths: torch.Tensor, - compress_ratio: int, - num_columns: int | None = None, - score_scale: float | None = None, -) -> tuple[torch.Tensor, torch.Tensor]: - """Compute QSA scores directly from a paged compressed-key cache.""" - _validate_mqa(q) - if not q.is_cuda or not HAS_TRITON: - raise RuntimeError("paged QSA scoring requires a GPU and Triton") - if k_cache.ndim != 4 or k_cache.shape[2] != 1: - raise ValueError("QSA cache must be [pages, page_size, 1, head_dim]") - if k_cache.shape[3] != q.shape[2]: - raise ValueError("QSA query and cache dimensions must match") - if page_table.ndim != 2: - raise ValueError("QSA page table must be two-dimensional") - if q.shape[0] and (not all(k_cache.shape[:2]) or not all(page_table.shape)): - raise ValueError("QSA paged scoring cache and page table must be nonempty") - if token_to_req.shape != (q.shape[0],): - raise ValueError("QSA request mapping must match query rows") - if query_positions.shape != (q.shape[0],): - raise ValueError("QSA query positions must match query rows") - if sequence_lengths.shape != (page_table.shape[0],): - raise ValueError("QSA sequence lengths must match page-table requests") - if compress_ratio <= 0: - raise ValueError("QSA compression ratio must be positive") - score_divisor = math.sqrt(q.shape[2]) if score_scale is None else score_scale - if score_divisor <= 0: - raise ValueError("QSA score scale must be positive") - - capacity = page_table.shape[1] * k_cache.shape[1] - columns = capacity if num_columns is None else num_columns - if columns < 0: - raise ValueError("QSA score width must be non-negative") - logits = torch.empty((q.shape[0], columns), dtype=torch.float32, device=q.device) - visible_blocks = torch.empty(q.shape[0], dtype=torch.int32, device=q.device) - if not q.shape[0] or not columns: - return logits, visible_blocks - block_n = 32 - _qsa_mqa_paged_kernel[(q.shape[0], triton.cdiv(columns, block_n))]( - q, - k_cache, - page_table, - token_to_req, - query_positions, - sequence_lengths, - visible_blocks, - logits, - q.stride(0), - q.stride(1), - q.stride(2), - k_cache.stride(0), - k_cache.stride(1), - k_cache.stride(3), - page_table.stride(0), - page_table.stride(1), - logits.stride(0), - q.shape[0], - columns, - k_cache.shape[0], - page_table.shape[0], - float(score_divisor), - PAGE_SIZE=k_cache.shape[1], - PAGE_TABLE_WIDTH=page_table.shape[1], - NUM_HEADS=q.shape[1], - HEAD_DIM=q.shape[2], - BLOCK_N=block_n, - BLOCK_D=triton.next_power_of_2(q.shape[2]), - COMPRESS_RATIO=compress_ratio, - num_warps=4, - ) - return logits, visible_blocks - - -def expand_qsa_block_indices_cuda( - block_indices: torch.Tensor, - query_positions: torch.Tensor, - sequence_lengths: torch.Tensor, - token_to_req: torch.Tensor, - compress_ratio: int, - token_topk: int, - out: torch.Tensor | None = None, -) -> torch.Tensor: - """Expand compressed blocks and compact the causal tail of the open group.""" - if not block_indices.is_cuda or not HAS_TRITON: - raise RuntimeError("QSA index expansion requires a GPU and Triton") - if token_topk % compress_ratio: - raise ValueError("QSA token top-k must be divisible by compression ratio") - block_topk = token_topk // compress_ratio - output_width = token_topk + compress_ratio - 1 - if block_indices.shape != (query_positions.numel(), block_topk): - raise ValueError("QSA compressed top-k has an invalid shape") - if token_to_req.shape != query_positions.shape: - raise ValueError("QSA request mapping must match query positions") - if sequence_lengths.ndim != 1 or not sequence_lengths.shape[0]: - raise ValueError("QSA request sequence lengths must be nonempty") - if out is None: - out = torch.empty( - (block_indices.shape[0], output_width), - dtype=torch.int32, - device=block_indices.device, - ) - elif out.shape != (block_indices.shape[0], output_width): - raise ValueError("QSA expansion output has an invalid shape") - if not block_indices.shape[0]: - return out - column_block = 256 - _expand_qsa_indices_kernel[ - (block_indices.shape[0], triton.cdiv(output_width, column_block)) - ]( - block_indices, - query_positions, - sequence_lengths, - token_to_req, - out, - block_indices.stride(0), - block_indices.stride(1), - out.stride(0), - out.stride(1), - block_indices.shape[0], - sequence_lengths.shape[0], - BLOCK_TOPK=block_topk, - COMPRESS_RATIO=compress_ratio, - TOKEN_TOPK=token_topk, - OUTPUT_WIDTH=output_width, - COLUMN_BLOCK=column_block, - num_warps=4, - ) - return out - - -def qsa_select_paged_tokens( - q: torch.Tensor, - k_cache: torch.Tensor, - page_table: torch.Tensor, - token_to_req: torch.Tensor, - query_positions: torch.Tensor, - sequence_lengths: torch.Tensor, - token_topk: int, - compress_ratio: int, - out: torch.Tensor | None = None, -) -> torch.Tensor: - """Score, select, and expand QSA indices without host synchronization.""" - rows = q.shape[0] - output_width = token_topk + compress_ratio - 1 - if out is None: - out = torch.empty((rows, output_width), dtype=torch.int32, device=q.device) - if out.shape != (rows, output_width): - raise ValueError("QSA selection output has an invalid shape") - if not rows: - return out - - columns = page_table.shape[1] * k_cache.shape[1] - block_topk = token_topk // compress_ratio - rows_per_chunk = max(1, _LOGITS_WORKSPACE_BYTES // max(columns * 4, 1)) - chunk_rows = min(rows, rows_per_chunk) - blocks_buffer = torch.empty( - (chunk_rows, block_topk), dtype=torch.int32, device=q.device - ) - topk_workspace = torch.empty( - (_TOPK_WORKSPACE_BYTES,), dtype=torch.uint8, device=q.device - ) - for row_start in range(0, rows, rows_per_chunk): - row_end = min(row_start + rows_per_chunk, rows) - row_slice = slice(row_start, row_end) - logits, visible_blocks = qsa_mqa_paged( - q[row_slice], - k_cache, - page_table, - token_to_req[row_slice], - query_positions[row_slice], - sequence_lengths, - compress_ratio, - ) - blocks = blocks_buffer[: row_end - row_start] - use_cooperative_topk = ( - current_platform.is_cuda() - and blocks.shape[0] <= 32 - and logits.stride(0) % 4 == 0 - and current_platform.has_device_capability(90) - and not current_platform.is_device_capability_family(120) - ) - if use_cooperative_topk: - torch.ops._C.cooperative_topk( - logits, - visible_blocks, - blocks, - topk_workspace, - block_topk, - columns, - ) - elif current_platform.is_cuda(): - torch.ops._C.persistent_topk( - logits, - visible_blocks, - blocks, - topk_workspace, - block_topk, - columns, - ) - else: - ops.top_k_per_row_decode( - logits, - 1, - visible_blocks, - blocks, - blocks.shape[0], - logits.stride(0), - logits.stride(1), - block_topk, - ) - expand_qsa_block_indices_cuda( - blocks, - query_positions[row_slice], - sequence_lengths, - token_to_req[row_slice], - compress_ratio, - token_topk, - out[row_slice], - ) - return out - - -def qsa_sparse_paged_attention( - q: torch.Tensor, - k_cache: torch.Tensor, - v_cache: torch.Tensor, - logical_indices: torch.Tensor, - block_table: torch.Tensor, - token_to_req: torch.Tensor, - out: torch.Tensor | None = None, -) -> torch.Tensor: - """Run sparse GQA directly over paged BF16 K/V caches.""" - if not q.is_cuda or not HAS_TRITON: - raise RuntimeError("paged QSA sparse attention requires a GPU and Triton") - if q.ndim != 3 or k_cache.ndim != 4 or v_cache.shape != k_cache.shape: - raise ValueError("QSA sparse attention received invalid Q/K/V shapes") - if logical_indices.ndim != 2 or logical_indices.shape[0] != q.shape[0]: - raise ValueError("QSA indices must have one row per query") - if token_to_req.shape != (q.shape[0],) or block_table.ndim != 2: - raise ValueError("QSA sparse attention metadata has invalid shapes") - if not all(k_cache.shape[:3]) or not all(block_table.shape): - raise ValueError("QSA sparse attention cache and block table must be nonempty") - if logical_indices.shape[1] <= 0: - raise ValueError("QSA sparse attention requires a positive selection width") - if q.shape[2] != k_cache.shape[3] or q.shape[1] % k_cache.shape[2]: - raise ValueError("QSA sparse attention requires valid grouped-query heads") - head_dim = q.shape[2] - assert head_dim >= 16 and (head_dim & (head_dim - 1)) == 0 - assert q.dtype == k_cache.dtype == v_cache.dtype == torch.bfloat16 - assert logical_indices.dtype == block_table.dtype == torch.int32 - assert token_to_req.dtype == torch.int32 - assert q.device == k_cache.device == v_cache.device - assert q.device == logical_indices.device == block_table.device - assert q.device == token_to_req.device - assert q.stride(2) == k_cache.stride(3) == v_cache.stride(3) == 1 - assert logical_indices.stride(1) == block_table.stride(1) == 1 - assert token_to_req.stride(0) == 1 - if out is None: - out = torch.empty_like(q) - if out.shape != q.shape: - raise ValueError("QSA sparse output must match its query") - assert out.dtype == q.dtype and out.device == q.device - assert out.stride(2) == 1 - if not q.shape[0]: - return out - - group_size = q.shape[1] // k_cache.shape[2] - block_m = triton.next_power_of_2(group_size) - base_programs = q.shape[0] * k_cache.shape[2] - small_profile_limit = 8 if block_m <= 8 else 4 - - # Tuned on GB300 for the Qwen-Air TP1, TP2, and TP4 attention shapes. - # Narrow tiles favor decode; wide tiles improve throughput for prefill. - if base_programs <= small_profile_limit: - block_n, target_splits, partial_warps = 16, 64, 4 - elif base_programs < 32: - block_n, target_splits, partial_warps = 16, 32, 4 - elif base_programs <= 256: - block_n, target_splits, partial_warps = 64, 8, 2 - elif base_programs <= 512: - block_n, target_splits, partial_warps = 64, 4, 2 - else: - block_n, target_splits, partial_warps = 64, 1, 2 - # gfx942 and gfx950 have a 64 KiB LDS limit. One software-pipelining - # stage keeps the wide TP4 tile within that shared-memory budget. - partial_stages = 1 if current_platform.is_rocm() else 2 - - num_tiles = triton.cdiv(logical_indices.shape[1], block_n) - # Avoid empty splits when the selection width is smaller than the profile. - max_useful_splits = 1 << (num_tiles.bit_length() - 1) - num_splits = min(max_useful_splits, target_splits) - - # Split=1 writes output directly and compiles out all workspace accesses. - if num_splits == 1: - partial_output = out - partial_lse = out - else: - # FP32 partials preserve accuracy when merging independently normalized - # splits. - partial_output = torch.empty( - (num_splits, *q.shape), dtype=torch.float32, device=q.device - ) - partial_lse = torch.empty( - (num_splits, q.shape[0], q.shape[1]), - dtype=torch.float32, - device=q.device, - ) - - partial_grid = (q.shape[0], k_cache.shape[2], num_splits) - _qsa_sparse_paged_gqa_splitk_kernel[partial_grid]( - q, - k_cache, - v_cache, - logical_indices, - block_table, - token_to_req, - partial_output, - partial_lse, - out, - q.stride(0), - q.stride(1), - k_cache.stride(0), - k_cache.stride(1), - k_cache.stride(2), - v_cache.stride(0), - v_cache.stride(1), - v_cache.stride(2), - logical_indices.stride(0), - block_table.stride(0), - out.stride(0), - out.stride(1), - q.shape[0], - k_cache.shape[0], - block_table.shape[0], - TOPK=logical_indices.shape[1], - PAGE_SIZE=k_cache.shape[1], - PAGE_TABLE_WIDTH=block_table.shape[1], - GROUP_SIZE=group_size, - HEAD_DIM=q.shape[2], - NUM_QUERY_HEADS=q.shape[1], - NUM_SPLITS=num_splits, - NUM_TILES=num_tiles, - BLOCK_M=block_m, - BLOCK_N=block_n, - num_warps=partial_warps, - num_stages=partial_stages, - ) - if num_splits == 1: - return out - - _qsa_merge_splitk_kernel[(q.shape[0], q.shape[1])]( - partial_output, - partial_lse, - out, - out.stride(0), - out.stride(1), - q.shape[0], - HEAD_DIM=q.shape[2], - NUM_QUERY_HEADS=q.shape[1], - NUM_SPLITS=num_splits, - BLOCK_SPLITS=triton.next_power_of_2(num_splits), - num_warps=2, - num_stages=1, - ) - return out - - -def qsa_store_cache_rows( - cache: torch.Tensor, - slot_mapping: torch.Tensor, - rows: torch.Tensor, -) -> None: - """Store fixed-width rows in a QSA cache without boolean indexing.""" - if not cache.is_cuda or not HAS_TRITON: - raise RuntimeError("QSA cache stores require a GPU and Triton") - if cache.ndim != 4 or cache.shape[2] != 1: - raise ValueError("QSA cache must be [pages, page_size, 1, width]") - if not all(cache.shape): - raise ValueError("QSA cache dimensions must be nonzero") - if rows.ndim == 3: - if rows.shape[1] != 1: - raise ValueError("QSA cache rows must have one head") - rows = rows[:, 0] - if rows.shape != (slot_mapping.numel(), cache.shape[3]): - raise ValueError("QSA cache rows and slots have incompatible shapes") - if not rows.shape[0]: - return - _store_qsa_rows_kernel[(rows.shape[0],)]( - cache, - slot_mapping, - rows, - cache.stride(0), - cache.stride(1), - cache.stride(3), - rows.stride(0), - rows.stride(1), - rows.shape[0], - cache.shape[0], - PAGE_SIZE=cache.shape[1], - WIDTH=cache.shape[3], - BLOCK_D=triton.next_power_of_2(cache.shape[3]), - num_warps=4, - ) - - -def qsa_compress_groups_with_ratio( - raw_keys: torch.Tensor, # this step's raw key rows [rows, 1, head_size] - raw_positions: torch.Tensor, # this step's positions [rows, 1, 3] int64 - compressor_state_cache: torch.Tensor, - compressor_state_block_table: torch.Tensor, - token_to_req: torch.Tensor, - query_start_loc: torch.Tensor, - logical_positions: torch.Tensor, - compressed_slots: torch.Tensor, - compress_ratio: int, - rope_cache: torch.Tensor | None = None, -) -> tuple[torch.Tensor, torch.Tensor]: - """Pool completed groups from the compressor-state ring and raw token rows.""" - if not raw_keys.is_cuda or not HAS_TRITON: - raise RuntimeError("QSA compression requires a GPU and Triton") - rows = token_to_req.numel() - if compress_ratio <= 0: - raise ValueError("QSA compression ratio must be positive") - if raw_keys.ndim != 3 or raw_keys.shape[:2] != (rows, 1): - raise ValueError("QSA raw keys must be [rows, 1, head_size]") - if raw_positions.shape != (rows, 1, 3) or raw_positions.dtype != torch.int64: - raise ValueError("QSA raw positions must be [rows, 1, 3] int64") - if logical_positions.shape != (rows,) or compressed_slots.shape != (rows,): - raise ValueError("QSA compression metadata must match token rows") - if compressor_state_cache.ndim != 4 or compressor_state_cache.shape[2] != 1: - raise ValueError("QSA compressor-state cache has an invalid shape") - if ( - # The ring is wider than one group so speculative rows cannot alias - # onto the committed keys of the group still being collected. - compressor_state_cache.shape[1] < compress_ratio - or compressor_state_cache.shape[3] != raw_keys.shape[2] - or compressor_state_cache.dtype != raw_keys.dtype - ): - raise ValueError( - "QSA compressor-state cache does not match the compression layout" - ) - if ( - compressor_state_block_table.ndim != 2 - or compressor_state_block_table.shape[1] < 1 - ): - raise ValueError( - "QSA compressor-state block table must contain one block per request" - ) - if query_start_loc.ndim != 1 or query_start_loc.shape[0] < 2: - raise ValueError("QSA query starts must contain a terminal offset") - num_requests = query_start_loc.shape[0] - 1 - if compressor_state_block_table.shape[0] < num_requests: - raise ValueError("QSA compressor-state block table has too few request rows") - if rope_cache is not None and ( - rope_cache.ndim != 4 - or rope_cache.shape[:3] != compressor_state_cache.shape[:3] - or rope_cache.shape[3] != 3 - or rope_cache.dtype != torch.int64 - ): - raise ValueError("QSA packed position view has an invalid shape or dtype") - if rows and ( - not all(compressor_state_cache.shape) - or not all(compressor_state_block_table.shape) - ): - raise ValueError("QSA compressor-state cache and block table must be nonempty") - pooled = torch.empty( - (rows, 1, raw_keys.shape[2]), - dtype=raw_keys.dtype, - device=raw_keys.device, - ) - first_positions = torch.empty((rows, 3), dtype=torch.int64, device=raw_keys.device) - if not rows: - return pooled, first_positions - if rope_cache is None: - rope_cache = compressor_state_cache - load_rope_positions = False - else: - load_rope_positions = True - _compress_qsa_groups_kernel[(rows,)]( - raw_keys, - raw_positions, - compressor_state_cache, - rope_cache, - compressor_state_block_table, - token_to_req, - query_start_loc, - logical_positions, - compressed_slots, - pooled, - first_positions, - raw_keys.stride(0), - raw_keys.stride(2), - raw_positions.stride(0), - raw_positions.stride(2), - compressor_state_cache.stride(0), - compressor_state_cache.stride(1), - compressor_state_cache.stride(3), - rope_cache.stride(0), - rope_cache.stride(1), - rope_cache.stride(3), - compressor_state_block_table.stride(0), - pooled.stride(0), - pooled.stride(2), - first_positions.stride(0), - first_positions.stride(1), - rows, - compressor_state_cache.shape[0], - num_requests, - COMPRESSOR_STATE_SIZE=compressor_state_cache.shape[1], - COMPRESS_RATIO=compress_ratio, - HEAD_DIM=raw_keys.shape[2], - LOAD_ROPE_POSITIONS=load_rope_positions, - BLOCK_D=triton.next_power_of_2(raw_keys.shape[2]), - num_warps=4, - ) - return pooled, first_positions - - -__all__ = [ - "expand_qsa_block_indices_cuda", - "qsa_compress_groups_with_ratio", - "qsa_mqa_paged", - "qsa_select_paged_tokens", - "qsa_sparse_paged_attention", - "qsa_store_cache_rows", -] diff --git a/vllm/models/qwen4_exp/amd/ops/qsa_prepare.py b/vllm/models/qwen4_exp/amd/ops/qsa_prepare.py deleted file mode 100644 index bc2fee3b3a..0000000000 --- a/vllm/models/qwen4_exp/amd/ops/qsa_prepare.py +++ /dev/null @@ -1,703 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Fused QSA prepare kernel for the AMD Qwen4Exp path. - -Started as a copy of the NVIDIA kernel (``nvidia/ops/qsa_prepare.py``) and is -kept separate so either vendor can change its own copy without re-validating -the other. The state source select is two masked loads rather than a pointer -select, which ROCm Triton rejects. -""" - -import torch -from torch import nn - -from vllm.triton_utils import tl, triton - - -def supports_fused_pre_indexer( - rotary_emb: nn.Module, - head_dim: int, - num_kv_heads: int, - compress_ratio: int, -) -> bool: - """Report whether this indexer's shapes match the fused kernel's assumptions. - - The kernel hard-codes the rotary layout and the single-KV-head group - compression it was written for; everything it rejects has a working unfused - path. - """ - rotary_dim = int(rotary_emb.rotary_dim) - mrope_section = getattr(rotary_emb, "mrope_section", None) - return ( - bool(getattr(rotary_emb, "is_neox_style", False)) - and ( - not mrope_section - or ( - len(mrope_section) == 3 - and sum(mrope_section) == rotary_dim // 2 - and bool(getattr(rotary_emb, "mrope_interleaved", False)) - ) - ) - and head_dim == 128 - and rotary_dim == 64 - and num_kv_heads == 1 - and compress_ratio > 1 - and compress_ratio & (compress_ratio - 1) == 0 - ) - - -@triton.jit -def _norm_rope( - x, - pos_t, - pos_h, - pos_w, - cos_sin_ptr, - cos_sin_stride, - norm_weight_ptr, - eps, - IS_MROPE: tl.constexpr, - MROPE_H: tl.constexpr, - MROPE_W: tl.constexpr, -): - """Apply Gemma RMSNorm and selected-axis NeoX RoPE to register rows.""" - TILE_T: tl.constexpr = x.shape[0] - TILE_H: tl.constexpr = x.shape[1] - D: tl.constexpr = x.shape[2] - ROWS: tl.constexpr = TILE_T * TILE_H - HALF: tl.constexpr = D // 2 - QUARTER: tl.constexpr = D // 4 - pairs = tl.arange(0, QUARTER) - if IS_MROPE: - # Qwen interleaves temporal, height, and width rotary pairs. Each axis - # still indexes the same position-major cos/sin table. - h_mask = ((pairs % 3) == 1) & (pairs <= 3 * MROPE_H) - w_mask = ((pairs % 3) == 2) & (pairs <= 3 * MROPE_W) - t_mask = ~(h_mask | w_mask) - base = cos_sin_ptr + pairs[None, :] - pos_rows = (pos_t, pos_h, pos_w) - axis_masks = (t_mask, h_mask, w_mask) - cos = tl.zeros((TILE_T, QUARTER), dtype=cos_sin_ptr.dtype.element_ty) - sin = tl.zeros((TILE_T, QUARTER), dtype=cos_sin_ptr.dtype.element_ty) - for axis in tl.static_range(3): - cos += tl.load( - base + pos_rows[axis][:, None] * cos_sin_stride, - mask=axis_masks[axis][None, :], - other=0, - ) - sin += tl.load( - base + pos_rows[axis][:, None] * cos_sin_stride + QUARTER, - mask=axis_masks[axis][None, :], - other=0, - ) - else: - cos = tl.load(cos_sin_ptr + pos_t[:, None] * cos_sin_stride + pairs[None, :]) - sin = tl.load( - cos_sin_ptr + pos_t[:, None] * cos_sin_stride + QUARTER + pairs[None, :] - ) - - cos = tl.reshape( - tl.broadcast_to(cos[:, None, :], (TILE_T, TILE_H, QUARTER)), - (ROWS, QUARTER), - ) - sin = tl.reshape( - tl.broadcast_to(sin[:, None, :], (TILE_T, TILE_H, QUARTER)), - (ROWS, QUARTER), - ) - x = tl.reshape(x, (ROWS, D)).to(tl.float32) - weight = tl.load(norm_weight_ptr + tl.arange(0, D)).to(tl.float32) + 1.0 - rrms = tl.rsqrt(tl.sum(x * x, axis=1) / D + eps) - y = (x * rrms[:, None] * weight[None, :]).to(cos.dtype) - rotated, passthrough = tl.split( - tl.permute(tl.reshape(y, (ROWS, 2, HALF)), (0, 2, 1)) - ) - r0, r1 = tl.split(tl.permute(tl.reshape(rotated, (ROWS, 2, QUARTER)), (0, 2, 1))) - out0 = r0 * cos - r1 * sin - out1 = r1 * cos + r0 * sin - rotated = tl.reshape(tl.permute(tl.join(out0, out1), (0, 2, 1)), (ROWS, HALF)) - result = tl.reshape(tl.permute(tl.join(rotated, passthrough), (0, 2, 1)), (ROWS, D)) - return tl.reshape(result, (TILE_T, TILE_H, D)) - - -@triton.jit -def _to_dst_dtype(x, dst, scale): - """Round to BF16 like the unfused path, then scale for an FP8 destination.""" - out_ty = dst.dtype.element_ty - x = x.to(tl.bfloat16) - if out_ty == tl.float8e4nv: - x = x.to(tl.float32) / scale - return x.to(out_ty) - - -@triton.jit -def _store_rotated(dst, y, o1, o2, scale): - """Store a normalized head whose first ``2 * len(o1)`` dims are rotated.""" - HALF: tl.constexpr = o1.shape[0] - dims = tl.arange(0, y.shape[0]) - rot = tl.arange(0, HALF) - tl.store(dst + dims, _to_dst_dtype(y, dst, scale), mask=dims >= 2 * HALF) - tl.store(dst + rot, _to_dst_dtype(o1, dst, scale)) - tl.store(dst + HALF + rot, _to_dst_dtype(o2, dst, scale)) - - -@triton.jit( - do_not_specialize=[ - "num_tokens", - "num_state_blocks", - "num_compressed_blocks", - "num_k_work", - ] -) -def _qsa_prepare_kernel( - q_ptr, - q_stride_token, - k_ptr, - k_stride_token, - pos_ptr, - pos_stride_axis, - pos_stride_token, - cos_sin_ptr, - q_norm_weight_ptr, - k_norm_weight_ptr, - eps, - q_out_ptr, - q_out_stride_token, - q_out_stride_head, - state_cache_ptr, - state_cache_stride_block, - state_cache_stride_token, - state_slots_ptr, - state_table_ptr, - state_table_stride_req, - query_start_loc_ptr, - logical_positions_ptr, - compressed_slots_ptr, - k_work_metadata_ptr, - compressed_cache_ptr, - compressed_cache_stride_block, - compressed_cache_stride_token, - num_tokens, - num_state_blocks, - num_compressed_blocks, - num_k_work, - HQ: tl.constexpr, - D: tl.constexpr, - TILE_T_Q: tl.constexpr, - TILE_H_Q: tl.constexpr, - COMPRESS_RATIO: tl.constexpr, - STATE_SIZE: tl.constexpr, - COMP_PAGE_SIZE: tl.constexpr, - IS_2D_POSITIONS: tl.constexpr, - IS_K_MROPE: tl.constexpr, - CACHE_HAS_ROPE_POS: tl.constexpr, - MROPE_H: tl.constexpr, - MROPE_W: tl.constexpr, - main_qkv_ptr, - main_q_norm_weight_ptr, - main_k_norm_weight_ptr, - main_eps, - main_q_out_ptr, - main_gate_out_ptr, - main_cache_ptr, - main_cache_stride_block, - main_cache_stride_token, - main_cache_stride_head, - main_slots_ptr, - main_k_scale, - main_v_scale, - MAIN_HQ: tl.constexpr, - MAIN_HK: tl.constexpr, - MAIN_D: tl.constexpr, - MAIN_PAGE_SIZE: tl.constexpr, -): - pid = tl.program_id(0) - num_index_work = num_k_work + tl.cdiv(num_tokens, TILE_T_Q) * tl.cdiv(HQ, TILE_H_Q) - if pid >= num_index_work: - # Main attention: one program per (token, Q or KV head) after the - # indexer work. RoPE covers the first D // 2 dims, the width of the - # shared cos/sin table. - main_pid = pid - num_index_work - token = (main_pid // (MAIN_HQ + MAIN_HK)).to(tl.int64) - head = main_pid % (MAIN_HQ + MAIN_HK) - HALF: tl.constexpr = D // 4 - dims = tl.arange(0, MAIN_D) - rot = tl.arange(0, HALF) - row = main_qkv_ptr + token * (2 * (MAIN_HQ + MAIN_HK) * MAIN_D) - is_k = head >= MAIN_HQ - kv_head = head - MAIN_HQ - if is_k: - src = row + (2 * MAIN_HQ + kv_head) * MAIN_D - weight_ptr = main_k_norm_weight_ptr - else: - src = row + 2 * head * MAIN_D - weight_ptr = main_q_norm_weight_ptr - x = tl.load(src + dims).to(tl.float32) - inv_rms = tl.rsqrt(tl.sum(x * x, axis=0) / MAIN_D + main_eps) - w = tl.load(weight_ptr + dims).to(tl.float32) + 1.0 - y = x * inv_rms * w - x1 = tl.load(src + rot).to(tl.float32) - x2 = tl.load(src + HALF + rot).to(tl.float32) - w1 = tl.load(weight_ptr + rot).to(tl.float32) + 1.0 - w2 = tl.load(weight_ptr + HALF + rot).to(tl.float32) + 1.0 - x1 = (x1 * inv_rms * w1).to(tl.bfloat16).to(tl.float32) - x2 = (x2 * inv_rms * w2).to(tl.bfloat16).to(tl.float32) - pos = tl.load(pos_ptr + token * pos_stride_token).to(tl.int64) - if IS_2D_POSITIONS: - pos_h = tl.load(pos_ptr + pos_stride_axis + token * pos_stride_token) - pos_w = tl.load(pos_ptr + 2 * pos_stride_axis + token * pos_stride_token) - is_h = (rot % 3 == 1) & (rot < 3 * MROPE_H) - is_w = (rot % 3 == 2) & (rot < 3 * MROPE_W) - pos = tl.where( - is_h, pos_h.to(tl.int64), tl.where(is_w, pos_w.to(tl.int64), pos) - ) - cos = tl.load(cos_sin_ptr + pos * (D // 2) + rot).to(tl.float32) - sin = tl.load(cos_sin_ptr + pos * (D // 2) + HALF + rot).to(tl.float32) - o1 = x1 * cos - x2 * sin - o2 = x2 * cos + x1 * sin - if is_k: - slot = tl.load(main_slots_ptr + token).to(tl.int64) - if slot >= 0: - dst = ( - main_cache_ptr - + (slot // MAIN_PAGE_SIZE) * main_cache_stride_block - + (slot % MAIN_PAGE_SIZE) * main_cache_stride_token - + kv_head * main_cache_stride_head - ) - _store_rotated(dst, y, o1, o2, main_k_scale) - v = tl.load(src + MAIN_HK * MAIN_D + dims) - tl.store(dst + MAIN_D + dims, _to_dst_dtype(v, dst, main_v_scale)) - else: - out = (token * MAIN_HQ + head) * MAIN_D - _store_rotated(main_q_out_ptr + out, y, o1, o2, None) - gate = tl.load(src + MAIN_D + dims) - tl.store(main_gate_out_ptr + out + dims, gate) - return - # K work occupies the first programs; the remaining programs tile Q. This - # keeps both paths in one launch while leaving their register shapes - # independent. - if pid >= num_k_work: - q_pid = pid - num_k_work - num_head_tiles: tl.constexpr = tl.cdiv(HQ, TILE_H_Q) - token_tile = q_pid // num_head_tiles - head_tile = q_pid % num_head_tiles - tokens = (token_tile * TILE_T_Q + tl.arange(0, TILE_T_Q)).to(tl.int64) - heads = head_tile * TILE_H_Q + tl.arange(0, TILE_H_Q) - valid_tokens = tokens < num_tokens - valid_heads = heads < HQ - dims = tl.arange(0, D) - mask = valid_tokens[:, None, None] & valid_heads[None, :, None] - x = tl.load( - q_ptr - + tokens[:, None, None] * q_stride_token - + heads[None, :, None] * D - + dims[None, None, :], - mask=mask, - other=0.0, - ) - pos_t = tl.load(pos_ptr + tokens * pos_stride_token, mask=valid_tokens, other=0) - if IS_2D_POSITIONS: - pos_h = tl.load( - pos_ptr + pos_stride_axis + tokens * pos_stride_token, - mask=valid_tokens, - other=0, - ) - pos_w = tl.load( - pos_ptr + 2 * pos_stride_axis + tokens * pos_stride_token, - mask=valid_tokens, - other=0, - ) - else: - pos_h = pos_t - pos_w = pos_t - y = _norm_rope( - x, - pos_t, - pos_h, - pos_w, - cos_sin_ptr, - D // 2, - q_norm_weight_ptr, - eps, - IS_2D_POSITIONS, - MROPE_H, - MROPE_W, - ) - tl.store( - q_out_ptr - + tokens[:, None, None] * q_out_stride_token - + heads[None, :, None] * q_out_stride_head - + dims[None, None, :], - y, - mask=mask, - ) - - if pid < num_k_work: - # One work item owns one completed compression group. Work item zero - # additionally commits the request's current raw-K suffix below. - work_metadata = tl.load(k_work_metadata_ptr + pid * 2 + tl.arange(0, 2)) - request, work_in_request = tl.split(work_metadata) - if request < 0: - return - - query_start = tl.load(query_start_loc_ptr + request) - query_end = tl.load(query_start_loc_ptr + request + 1) - query_len = query_end - query_start - chunk_end = tl.load(logical_positions_ptr + query_end - 1) - chunk_start = chunk_end - query_len + 1 - num_groups = (chunk_end + 1) // COMPRESS_RATIO - chunk_start // COMPRESS_RATIO - dims = tl.arange(0, D) - - if work_in_request < num_groups: - first_boundary = ( - (chunk_start + COMPRESS_RATIO) // COMPRESS_RATIO - ) * COMPRESS_RATIO - 1 - end_position = first_boundary + work_in_request * COMPRESS_RATIO - boundary_token = query_start + end_position - chunk_start - valid_token = ( - (boundary_token >= query_start) - & (boundary_token < query_end) - & (boundary_token < num_tokens) - ) - compressed_slot = tl.load( - compressed_slots_ptr + boundary_token, - mask=valid_token, - other=-1, - ) - valid = ( - valid_token - & (compressed_slot >= 0) - & (compressed_slot < num_compressed_blocks * COMP_PAGE_SIZE) - ) - state_block = tl.load(state_table_ptr + request * state_table_stride_req) - state_block_valid = (state_block >= 0) & (state_block < num_state_blocks) - safe_state_block = tl.maximum(state_block, 0).to(tl.int64) - group_offsets = tl.arange(0, COMPRESS_RATIO) - source_positions = end_position - (COMPRESS_RATIO - 1) + group_offsets - source_in_chunk = source_positions >= chunk_start - source_in_cache = source_positions < chunk_start - source_tokens = query_start + source_positions - chunk_start - source_tokens_valid = ( - (source_tokens >= query_start) - & (source_tokens < query_end) - & (source_tokens < num_tokens) - ) - current_base = ( - k_ptr - + tl.maximum(source_tokens, 0).to(tl.int64)[:, None] * k_stride_token - ) - cached_base = ( - state_cache_ptr - + safe_state_block * state_cache_stride_block - + (source_positions % STATE_SIZE)[:, None] * state_cache_stride_token - ) - # Only the first completed group can cross the chunk boundary, so - # every row comes from exactly one of the two sources. Selecting - # between the base pointers would need a pointer-typed tl.where, - # which ROCm's Triton rejects ("'tt.addptr' op failed to verify - # that result type matches ptr type"), so select the loaded values - # instead. The two masks are complementary, so no lane fetches from - # both and memory traffic is unchanged. - # - # Both bases still need the alignment stated explicitly. Without it - # Triton vectorizes these loads differently, which reorders the - # pooling reduction below; float addition is not associative, so the - # compressed rows stop matching the unfused path bit for bit and an - # FP8 cache rounds a near-zero pooled value to the wrong code. - current_base = tl.multiple_of(current_base, (8, 8)) - cached_base = tl.multiple_of(cached_base, (8, 8)) - from_chunk = valid & source_in_chunk[:, None] & source_tokens_valid[:, None] - from_cache = valid & source_in_cache[:, None] & state_block_valid - current = tl.load(current_base + dims[None, :], mask=from_chunk, other=0.0) - cached = tl.load(cached_base + dims[None, :], mask=from_cache, other=0.0) - source = tl.where(source_in_chunk[:, None], current, cached).to(tl.float32) - # Match the unfused path's BF16 pooled tensor before RMSNorm. - pooled = ( - (tl.sum(source, axis=0) / COMPRESS_RATIO).to(tl.bfloat16).to(tl.float32) - ) - - first_position = end_position - (COMPRESS_RATIO - 1) - if CACHE_HAS_ROPE_POS: - # RoPE uses the first token in the pooled group. Its exact MRoPE - # coordinates may live in this chunk or the raw-state ring. - first_in_chunk = first_position >= chunk_start - first_token = query_start + first_position - chunk_start - first_token_valid = ( - (first_token >= query_start) - & (first_token < query_end) - & (first_token < num_tokens) - ) - safe_first_token = tl.maximum(first_token, 0) - load_current_position = first_in_chunk & first_token_valid - first_pos_t = tl.load( - pos_ptr + safe_first_token * pos_stride_token, - mask=load_current_position, - other=0, - ) - if IS_2D_POSITIONS: - first_pos_h = tl.load( - pos_ptr + pos_stride_axis + safe_first_token * pos_stride_token, - mask=load_current_position, - other=0, - ) - first_pos_w = tl.load( - pos_ptr - + 2 * pos_stride_axis - + safe_first_token * pos_stride_token, - mask=load_current_position, - other=0, - ) - else: - first_pos_h = first_pos_t - first_pos_w = first_pos_t - tail = ( - state_cache_ptr - + safe_state_block * state_cache_stride_block - + (first_position % STATE_SIZE) * state_cache_stride_token - + D - ).to(tl.pointer_type(tl.int64)) - load_cached_position = ~first_in_chunk & state_block_valid - cached_pos_t = tl.load(tail, mask=load_cached_position, other=0) - cached_pos_h = tl.load(tail + 1, mask=load_cached_position, other=0) - cached_pos_w = tl.load(tail + 2, mask=load_cached_position, other=0) - pos_t = tl.where(first_in_chunk, first_pos_t.to(tl.int64), cached_pos_t) - pos_h = tl.where(first_in_chunk, first_pos_h.to(tl.int64), cached_pos_h) - pos_w = tl.where(first_in_chunk, first_pos_w.to(tl.int64), cached_pos_w) - else: - pos_t = first_position - pos_h = first_position - pos_w = first_position - y = _norm_rope( - tl.reshape(pooled, (1, 1, D)), - pos_t + tl.arange(0, 1), - pos_h + tl.arange(0, 1), - pos_w + tl.arange(0, 1), - cos_sin_ptr, - D // 2, - k_norm_weight_ptr, - eps, - IS_K_MROPE, - MROPE_H, - MROPE_W, - ) - compressed_block = (compressed_slot // COMP_PAGE_SIZE).to(tl.int64) - compressed_row = compressed_slot % COMP_PAGE_SIZE - tl.store( - compressed_cache_ptr - + compressed_block * compressed_cache_stride_block - + compressed_row * compressed_cache_stride_token - + dims, - tl.reshape(y, (D,)), - mask=valid, - ) - - if work_in_request == 0: - # This CTA may have just read historical rows from the circular buffer. - # Keep every lane past those loads before overwriting the same ring. - tl.debug_barrier() - # One CTA per request commits only the suffix retained by the ring. - num_state_rows = tl.minimum(query_len, STATE_SIZE) - for state_offset in tl.range(0, num_state_rows): - token = (query_end - num_state_rows + state_offset).to(tl.int64) - valid_token = ( - (token >= query_start) & (token < query_end) & (token < num_tokens) - ) - slot = tl.load(state_slots_ptr + token, mask=valid_token, other=-1) - valid_slot = ( - valid_token & (slot >= 0) & (slot < num_state_blocks * STATE_SIZE) - ) - safe_slot = tl.maximum(slot, 0) - state_row = ( - state_cache_ptr - + (safe_slot // STATE_SIZE).to(tl.int64) * state_cache_stride_block - + (safe_slot % STATE_SIZE) * state_cache_stride_token - ) - k = tl.load( - k_ptr + tl.maximum(token, 0) * k_stride_token + dims, - mask=valid_slot, - other=0.0, - ) - tl.store(state_row + dims, k, mask=valid_slot) - if CACHE_HAS_ROPE_POS: - pos_t = tl.load( - pos_ptr + tl.maximum(token, 0) * pos_stride_token, - mask=valid_slot, - other=0, - ) - if IS_2D_POSITIONS: - pos_h = tl.load( - pos_ptr - + pos_stride_axis - + tl.maximum(token, 0) * pos_stride_token, - mask=valid_slot, - other=0, - ) - pos_w = tl.load( - pos_ptr - + 2 * pos_stride_axis - + tl.maximum(token, 0) * pos_stride_token, - mask=valid_slot, - other=0, - ) - else: - pos_h = pos_t - pos_w = pos_t - tail = (state_row + D).to(tl.pointer_type(tl.int64)) - tl.store(tail, pos_t.to(tl.int64), mask=valid_slot) - tl.store(tail + 1, pos_h.to(tl.int64), mask=valid_slot) - tl.store(tail + 2, pos_w.to(tl.int64), mask=valid_slot) - - -def qsa_prepare( - q: torch.Tensor, - k: torch.Tensor, - positions: torch.Tensor, - cos_sin_cache: torch.Tensor, - q_norm_weight: torch.Tensor, - k_norm_weight: torch.Tensor, - eps: float, - q_out: torch.Tensor, - state_cache: torch.Tensor, - state_slots: torch.Tensor, - state_block_table: torch.Tensor, - query_start_loc: torch.Tensor, - logical_positions: torch.Tensor, - compressed_cache: torch.Tensor, - compressed_slots: torch.Tensor, - k_work_metadata: torch.Tensor, - *, - compress_ratio: int, - mrope_section: tuple[int, int, int] | None, - rope_pos_offset: int | None, - main_qkv: torch.Tensor, - main_q_norm_weight: torch.Tensor, - main_k_norm_weight: torch.Tensor, - main_eps: float, - main_kv_cache: torch.Tensor, - main_slot_mapping: torch.Tensor, - main_k_scale: float, - main_v_scale: float, - main_gate_out: torch.Tensor | None = None, -) -> tuple[torch.Tensor, torch.Tensor]: - """Normalize Q, compress K, then update the circular raw state. - - Also prepares the main attention (QK-norm/RoPE, gate copy, K/V cache write) - and returns its Q and gate. ``main_gate_out`` lets the gate land in a - caller-owned buffer, e.g. one that outlives an opaque custom op. - """ - num_tokens = q.shape[0] - main_head_dim = main_kv_cache.shape[-1] // 2 - num_main_kv_heads = main_kv_cache.shape[2] - num_main_q_heads = main_qkv.shape[1] // (2 * main_head_dim) - num_main_kv_heads - main_q_out = main_qkv.new_empty(num_tokens, num_main_q_heads, main_head_dim) - if main_gate_out is None: - main_gate_out = torch.empty_like(main_q_out) - else: - assert main_gate_out.shape == main_q_out.shape - assert main_gate_out.is_contiguous() - if num_tokens == 0: - return main_q_out, main_gate_out - num_q_heads, head_dim = q_out.shape[1:] - assert cos_sin_cache.shape[-1] * 2 == head_dim - assert q.shape == (num_tokens, num_q_heads * head_dim) - assert k.shape == (num_tokens, head_dim) - assert q.stride(-1) == 1 - assert k.stride(-1) == 1 - assert q_out.stride(-1) == 1 - assert cos_sin_cache.is_contiguous() - assert state_cache.stride(-1) == 1 - assert compressed_cache.stride(-1) == 1 - assert k_work_metadata.ndim == 2 and k_work_metadata.shape[1] == 2 - is_2d_positions = positions.ndim == 2 - is_k_mrope = bool(mrope_section) - cache_has_rope_pos = rope_pos_offset is not None - assert rope_pos_offset is None or rope_pos_offset == head_dim - if is_2d_positions: - assert positions.shape == (3, num_tokens) - assert is_k_mrope - pos_stride_axis, pos_stride_token = positions.stride() - else: - assert positions.shape == (num_tokens,) - pos_stride_axis, pos_stride_token = 0, positions.stride(0) - section = mrope_section if mrope_section is not None else (0, 0, 0) - assert len(section) == 3 - qkv_width = 2 * (num_main_q_heads + num_main_kv_heads) * main_head_dim - assert main_qkv.shape == (num_tokens, qkv_width) and main_qkv.is_contiguous() - assert main_slot_mapping.shape == (num_tokens,) - - # Swept on MI355X (64-lane wavefronts) over TILE_T_Q 1-16, TILE_H_Q 1-4 and - # num_warps 1/2/4, from 4 to 16384 tokens: no config beat these by more - # than noise, and 2 or 4 warps were slower everywhere. - if num_tokens <= 4096: - TILE_T_Q, TILE_H_Q = 2, 2 - else: - TILE_T_Q, TILE_H_Q = 2, 4 - num_k_work = k_work_metadata.shape[0] - num_q_work = triton.cdiv(num_tokens, TILE_T_Q) * triton.cdiv(num_q_heads, TILE_H_Q) - num_main_work = num_tokens * (num_main_q_heads + num_main_kv_heads) - _qsa_prepare_kernel[(num_k_work + num_q_work + num_main_work,)]( - q, - q.stride(0), - k, - k.stride(0), - positions, - pos_stride_axis, - pos_stride_token, - cos_sin_cache, - q_norm_weight, - k_norm_weight, - eps, - q_out, - q_out.stride(0), - q_out.stride(1), - state_cache, - state_cache.stride(0), - state_cache.stride(1), - state_slots, - state_block_table, - state_block_table.stride(0), - query_start_loc, - logical_positions, - compressed_slots, - k_work_metadata, - compressed_cache, - compressed_cache.stride(0), - compressed_cache.stride(1), - num_tokens, - state_cache.shape[0], - compressed_cache.shape[0], - num_k_work, - HQ=num_q_heads, - D=head_dim, - TILE_T_Q=TILE_T_Q, - TILE_H_Q=TILE_H_Q, - COMPRESS_RATIO=compress_ratio, - STATE_SIZE=state_cache.shape[1], - COMP_PAGE_SIZE=compressed_cache.shape[1], - IS_2D_POSITIONS=is_2d_positions, - IS_K_MROPE=is_k_mrope, - CACHE_HAS_ROPE_POS=cache_has_rope_pos, - MROPE_H=section[1], - MROPE_W=section[2], - main_qkv_ptr=main_qkv, - main_q_norm_weight_ptr=main_q_norm_weight, - main_k_norm_weight_ptr=main_k_norm_weight, - main_eps=main_eps, - main_q_out_ptr=main_q_out, - main_gate_out_ptr=main_gate_out, - main_cache_ptr=main_kv_cache, - main_cache_stride_block=main_kv_cache.stride(0), - main_cache_stride_token=main_kv_cache.stride(1), - main_cache_stride_head=main_kv_cache.stride(2), - main_slots_ptr=main_slot_mapping, - main_k_scale=main_k_scale, - main_v_scale=main_v_scale, - MAIN_HQ=num_main_q_heads, - MAIN_HK=num_main_kv_heads, - MAIN_D=main_head_dim, - MAIN_PAGE_SIZE=main_kv_cache.shape[1], - num_warps=1, - ) - return main_q_out, main_gate_out - - -__all__ = ["qsa_prepare", "supports_fused_pre_indexer"] diff --git a/vllm/models/qwen4_exp/amd/ops/sequence_parallel.py b/vllm/models/qwen4_exp/amd/ops/sequence_parallel.py deleted file mode 100644 index 2efcd56d54..0000000000 --- a/vllm/models/qwen4_exp/amd/ops/sequence_parallel.py +++ /dev/null @@ -1,47 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -import torch - -from vllm.distributed import get_tensor_model_parallel_world_size -from vllm.models.common.ops import sequence_parallel as common_sp -from vllm.utils.math_utils import cdiv -from vllm.utils.torch_utils import direct_register_custom_op - - -def _sp_shard_op(x: torch.Tensor) -> torch.Tensor: - output = common_sp.sp_shard(x) - # A functional custom op must not return a view of its input. - unpadded = x.shape[0] % get_tensor_model_parallel_world_size() == 0 - return output.clone() if unpadded else output - - -def _sp_local_fake(x: torch.Tensor) -> torch.Tensor: - tp_size = get_tensor_model_parallel_world_size() - return x.new_empty((cdiv(x.shape[0], tp_size), *x.shape[1:])) - - -# Dynamo would specialize the padding branch on the traced token count, and -# vLLM drops the resulting guards. Keep the branch opaque when compiling. -direct_register_custom_op( - op_name="sp_reduce_scatter", - op_func=common_sp.sp_reduce_scatter, - fake_impl=_sp_local_fake, -) -direct_register_custom_op( - op_name="sp_shard", - op_func=_sp_shard_op, - fake_impl=_sp_local_fake, -) - - -def sp_reduce_scatter(x: torch.Tensor) -> torch.Tensor: - if torch.compiler.is_compiling(): - return torch.ops.vllm.sp_reduce_scatter(x) - return common_sp.sp_reduce_scatter(x) - - -def sp_shard(x: torch.Tensor) -> torch.Tensor: - if torch.compiler.is_compiling(): - return torch.ops.vllm.sp_shard(x) - return common_sp.sp_shard(x) diff --git a/vllm/models/qwen4_exp/amd/ple_layer.py b/vllm/models/qwen4_exp/amd/ple_layer.py deleted file mode 100644 index c1cfc0f246..0000000000 --- a/vllm/models/qwen4_exp/amd/ple_layer.py +++ /dev/null @@ -1,883 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""GPU-resident Qwen4Exp position-learning enhancement layers.""" - -from collections.abc import Iterable, Sequence - -import torch -from torch import nn -from transformers import Qwen4ExpTextConfig - -from vllm.config import CacheConfig, ModelConfig, VllmConfig, get_current_vllm_config -from vllm.distributed import get_etp_group, get_tp_group -from vllm.forward_context import get_forward_context -from vllm.logger import init_logger -from vllm.model_executor.layers.linear import MergedColumnParallelLinear -from vllm.model_executor.layers.mamba.abstract import MambaBase -from vllm.model_executor.layers.mamba.mamba_utils import ( - MambaStateDtypeCalculator, - MambaStateShapeCalculator, - is_conv_state_dim_first, -) -from vllm.model_executor.layers.quantization.base_config import ( - QuantizationConfig, -) -from vllm.model_executor.layers.quantization.utils.fp8_utils import is_fp8 -from vllm.model_executor.models.utils import AutoWeightsLoader -from vllm.models.common.ops.sequence_parallel import ( - sp_all_gather, - sp_reduce_scatter, - sp_shard, -) -from vllm.utils.torch_utils import direct_register_custom_op -from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum -from vllm.v1.attention.backends.short_conv_attn import ( - PleShortConvAttentionBackend, - PleShortConvAttentionMetadata, -) - -from ..common.ngram_embedding import ( - Qwen4ExpPLEDeviceEmbedding, - Qwen4ExpPLEEmbeddingMethod, - Qwen4ExpPLENvFp4EmbeddingMethod, - Qwen4ExpPLEPinnedHostEmbedding, -) -from ..common.ops.ple import ple_conv, ple_gate, ple_ngram_ids - -logger = init_logger(__name__) - - -class Qwen4ExpPLEGroupedNorm(nn.Module): - def __init__( - self, - hidden_size: int, - eps: float, - group_size: int | None, - dtype: torch.dtype | None, - ) -> None: - super().__init__() - if group_size is not None and hidden_size % group_size: - raise ValueError( - f"hidden_size ({hidden_size}) must be divisible by " - f"group_size ({group_size})" - ) - self.eps = eps - self.group_size = group_size - self.weight = nn.Parameter(torch.zeros(hidden_size, dtype=dtype)) - - def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: - input_dtype = hidden_states.dtype - hidden_states = hidden_states.float() - if self.group_size is None: - variance = hidden_states.square().mean(dim=-1, keepdim=True) - normalized = hidden_states * torch.rsqrt(variance + self.eps) - else: - grouped = hidden_states.unflatten( - -1, (hidden_states.shape[-1] // self.group_size, self.group_size) - ) - variance = grouped.square().mean(dim=-1, keepdim=True) - normalized = (grouped * torch.rsqrt(variance + self.eps)).flatten(-2) - return (normalized * (1.0 + self.weight.float())).to(input_dtype) - - -class Qwen4ExpNGramEmbedding(nn.Module): - _MASK64 = (1 << 64) - 1 - _SPLITMIX_GAMMA = 0x9E3779B97F4A7C15 - _SPLITMIX_M1 = 0xBF58476D1CE4E5B9 - _SPLITMIX_M2 = 0x94D049BB133111EB - _PLE_LAYER_PRIME = 10007 - - @classmethod - def _splitmix64(cls, value: int) -> int: - """Mix an integer into a deterministic unsigned 64-bit value.""" - value = (value + cls._SPLITMIX_GAMMA) & cls._MASK64 - value = ((value ^ (value >> 30)) * cls._SPLITMIX_M1) & cls._MASK64 - value = ((value ^ (value >> 27)) * cls._SPLITMIX_M2) & cls._MASK64 - return (value ^ (value >> 31)) & cls._MASK64 - - @staticmethod - def _is_prime_64(value: int) -> bool: - """Return whether a 64-bit integer is prime.""" - if value < 2: - return False - for prime in (2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37): - if value % prime == 0: - return value == prime - exponent = value - 1 - shifts = 0 - while exponent % 2 == 0: - exponent //= 2 - shifts += 1 - for base in (2, 325, 9375, 28178, 450775, 9780504, 1795265022): - if base % value == 0: - continue - witness = pow(base, exponent, value) - if witness in (1, value - 1): - continue - for _ in range(shifts - 1): - witness = pow(witness, 2, value) - if witness == value - 1: - break - else: - return False - return True - - @classmethod - def _nth_prime_after(cls, start: int, count: int) -> int: - """Return the ``count``-th prime strictly greater than ``start``.""" - prime = int(start) - for _ in range(count): - candidate = prime + 1 - if candidate <= 2: - prime = 2 - continue - if candidate % 2 == 0: - candidate += 1 - while not cls._is_prime_64(candidate): - candidate += 2 - prime = candidate - return prime - - @classmethod - def _make_layer_multipliers( - cls, - *, - ngram_size: int, - unigram_vocab_size: int, - seed: int, - ple_dense_layer_id: int, - ) -> list[int]: - """Build deterministic hash multipliers for one PLE layer.""" - max_multiplier = ((1 << 63) - 1) // unigram_vocab_size - half_bound = max(1, max_multiplier // 2) - base_seed = seed + cls._PLE_LAYER_PRIME * ple_dense_layer_id - multipliers = [] - for index in range(ngram_size): - value = base_seed + cls._SPLITMIX_GAMMA * (index + 1) - multipliers.append(2 * (cls._splitmix64(value) % half_bound) + 1) - return multipliers - - @classmethod - def _make_vocab_layout( - cls, - *, - ngram_vocab_size_base: int, - ngram_heads: int, - ple_dense_layer_id: int, - ) -> tuple[list[int], list[int], int]: - """Build per-head vocabulary sizes, offsets, and total row count.""" - sizes: list[int] = [] - offsets: list[int] = [] - offset = 0 - for local_head in range(ngram_heads): - global_head = ple_dense_layer_id * ngram_heads + local_head - size = cls._nth_prime_after(ngram_vocab_size_base - 1, global_head + 1) - sizes.append(size) - offsets.append(offset) - offset += size - return sizes, offsets, offset - - def __init__( - self, - config: Qwen4ExpTextConfig, - embedding_dim: int, - ple_dense_layer_id: int, - max_total_tokens: int, - prefix: str, - layer_name: str, - *, - data_parallel_rank: int = 0, - quant_config: QuantizationConfig | None = None, - params_dtype: torch.dtype | None = None, - use_sequence_parallel: bool = False, - ) -> None: - super().__init__() - self.use_sequence_parallel = use_sequence_parallel - self.embedding_dim = embedding_dim - self.layer_name = layer_name - self.ngram_size = int(config.ngram_size) - self.heads_per_ngram = int(config.heads_per_ngram) - self.ngram_heads = (self.ngram_size - 1) * self.heads_per_ngram - if self.ngram_size < 2: - raise ValueError(f"ngram_size must be >= 2, got {self.ngram_size}") - if self.heads_per_ngram <= 0: - raise ValueError(f"heads_per_ngram must be > 0, got {self.heads_per_ngram}") - if embedding_dim % self.ngram_heads: - raise ValueError( - "ple_embed_dim must be divisible by total ngram heads: " - f"{embedding_dim} % {self.ngram_heads} != 0" - ) - self.head_dim = embedding_dim // self.ngram_heads - self.eos_token_id = int(config.eos_token_id) - self.unigram_vocab_size = int(config.vocab_size) - self.split_ngram_parts = int(getattr(config, "split_ngram_parts", 512)) - if self.split_ngram_parts <= 0: - raise ValueError("split_ngram_parts must be positive") - - multipliers = self._make_layer_multipliers( - ngram_size=self.ngram_size, - unigram_vocab_size=self.unigram_vocab_size, - seed=int(getattr(config, "seed", 1234)), - ple_dense_layer_id=ple_dense_layer_id, - ) - self.register_buffer( - "layer_multipliers", - torch.tensor(multipliers, dtype=torch.long), - persistent=True, - ) - - sizes, offsets, total_vocab_size = self._make_vocab_layout( - ngram_vocab_size_base=int(config.ngram_vocab_size_base), - ngram_heads=self.ngram_heads, - ple_dense_layer_id=ple_dense_layer_id, - ) - self.register_buffer( - "ngram_heads_vocab_sizes", - torch.tensor(sizes, dtype=torch.long), - persistent=True, - ) - self.register_buffer( - "ngram_heads_offsets", - torch.tensor(offsets, dtype=torch.long), - persistent=True, - ) - divisor = int(config.make_ngram_vocab_size_divisible_by) - padded_vocab_size = ((total_vocab_size + divisor - 1) // divisor) * divisor - if params_dtype is None: - params_dtype = torch.get_default_dtype() - embedding_prefix = f"{prefix}.ngram_embedding" - embedding_quant_method = Qwen4ExpPLEEmbeddingMethod.from_quant_config( - quant_config, - embedding_prefix, - getattr(config, "ple_embedding_dtype", None), - ) - if isinstance(embedding_quant_method, Qwen4ExpPLENvFp4EmbeddingMethod): - raise NotImplementedError( - "NVFP4 Qwen4Exp PLE tables are not supported on ROCm yet; use a " - "checkpoint with a BF16 or FP8 PLE table." - ) - engram_config = get_current_vllm_config().engram_config - embedding_cls = ( - Qwen4ExpPLEPinnedHostEmbedding - if engram_config is not None and engram_config.cpu_offload - else Qwen4ExpPLEDeviceEmbedding - ) - # A TP reduce-scatter covers all vocabulary owners only when ETP=TP. - self.use_reduce_scatter = ( - self.use_sequence_parallel - and get_etp_group().world_size == get_tp_group().world_size - ) - self.ngram_embedding = embedding_cls( - padded_vocab_size, - self.head_dim, - params_dtype=params_dtype, - padding_size=divisor, - prefix=embedding_prefix, - embedding_method=embedding_quant_method, - num_ngram_heads=self.ngram_heads, - max_total_tokens=max_total_tokens, - data_parallel_rank=data_parallel_rank, - reduce_results=not self.use_reduce_scatter, - ) - logger.info( - "Initialized AMD PLE embedding %s: quantization_method=%s, " - "weight_dtype=%s, weight_device=%s, pinned=%s", - embedding_prefix, - type(embedding_quant_method).__name__, - self.ngram_embedding.weight.dtype, - self.ngram_embedding.weight.device, - self.ngram_embedding.weight.is_pinned(), - ) - - def _sp_shard_embeddings(self, embeddings: torch.Tensor) -> torch.Tensor: - """Shard embedding token rows, optionally reducing vocabulary contributions.""" - # Each embedding element has one vocabulary owner, including FP8 bytes. - output_dtype = embeddings.dtype - if is_fp8(embeddings): - embeddings = embeddings.view(torch.int8) - if self.use_reduce_scatter: - embeddings = sp_reduce_scatter(embeddings) - else: - embeddings = sp_shard(embeddings) - return embeddings.view(output_dtype) - - def forward( - self, - input_ids: torch.Tensor, - query_start_loc: torch.Tensor, - ngram_context: torch.Tensor, - ) -> torch.Tensor: - """Return full embeddings or a token shard after combining vocabulary owners.""" - input_ids = input_ids.reshape(-1) - ngram_ids = input_ids.new_empty( - (input_ids.shape[0], self.ngram_heads), dtype=torch.int64 - ) - torch.ops.vllm.qwen4_exp_amd_ple_ngram_ids( - input_ids, - query_start_loc, - ngram_context, - ngram_ids, - self.layer_name, - ) - embedding = self.ngram_embedding - if embedding.supports_prefetch: - output = ngram_ids.new_empty( - (ngram_ids.shape[0], self.embedding_dim), - dtype=embedding.weight.dtype, - ) - torch.ops.vllm.qwen4_exp_amd_ple_ngram_embedding_pinned( - ngram_ids, - output, - self.layer_name, - ) - else: - output = ngram_ids.new_empty( - (ngram_ids.shape[0], self.embedding_dim), - dtype=embedding.params_dtype, - ) - torch.ops.vllm.qwen4_exp_amd_ple_ngram_embedding( - ngram_ids, - output, - self.layer_name, - ) - if not self.use_sequence_parallel: - return output - return self._sp_shard_embeddings(output) - - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - """Load hash buffers and checkpoint-split embedding rows.""" - persistent_buffers = { - "layer_multipliers": self.layer_multipliers, - "ngram_heads_offsets": self.ngram_heads_offsets, - "ngram_heads_vocab_sizes": self.ngram_heads_vocab_sizes, - } - loaded: set[str] = set() - regular_weights: list[tuple[str, torch.Tensor]] = [] - shard_prefix = "ngram_embedding.shard_" - - for name, loaded_weight in weights: - leaf_name = name.rsplit(".", 1)[-1] - if leaf_name.startswith("hashstats_") or leaf_name == "token_lookup": - continue - if name in persistent_buffers: - buffer = persistent_buffers[name] - if buffer.shape != loaded_weight.shape: - raise ValueError( - f"Shape mismatch for {name}: expected " - f"{tuple(buffer.shape)}, got {tuple(loaded_weight.shape)}" - ) - buffer.copy_(loaded_weight.to(device=buffer.device, dtype=buffer.dtype)) - loaded.add(name) - continue - if name.startswith(shard_prefix) and name.endswith(".weight"): - shard_text = name[len(shard_prefix) : -len(".weight")] - if not shard_text.isdigit(): - regular_weights.append((name, loaded_weight)) - continue - shard_index = int(shard_text) - if shard_index >= self.split_ngram_parts: - raise ValueError( - f"PLE embedding shard index {shard_index} exceeds " - f"split_ngram_parts={self.split_ngram_parts}" - ) - embedding = self.ngram_embedding - shard_size = ( - embedding.org_vocab_size + self.split_ngram_parts - 1 - ) // self.split_ngram_parts - checkpoint_start = shard_index * shard_size - expected_rows = max( - 0, - min(shard_size, embedding.org_vocab_size - checkpoint_start), - ) - expected_shape = (expected_rows, embedding.embedding_dim) - if tuple(loaded_weight.shape) != expected_shape: - raise ValueError( - f"Shape mismatch for PLE embedding shard {shard_index}: " - f"expected {expected_shape}, got " - f"{tuple(loaded_weight.shape)}" - ) - embedding.weight.weight_loader( - embedding.weight, - loaded_weight, - checkpoint_start=checkpoint_start, - ) - loaded.add("ngram_embedding.weight") - continue - regular_weights.append((name, loaded_weight)) - - if regular_weights: - loaded.update(AutoWeightsLoader(self).load_weights(regular_weights)) - return loaded - - -class Qwen4ExpPLELayer(nn.Module, MambaBase): - def __init__( - self, - config: Qwen4ExpTextConfig, - vllm_config: VllmConfig, - layer_idx: int = 0, - ple_dense_layer_id: int | None = None, - prefix: str = "", - use_sequence_parallel: bool = False, - ) -> None: - super().__init__() - self.use_sequence_parallel = use_sequence_parallel - model_config = vllm_config.model_config - cache_config = vllm_config.cache_config - quant_config = vllm_config.quant_config - self.model_config: ModelConfig = model_config - self.cache_config: CacheConfig = cache_config - self.layer_idx = layer_idx - self.ple_dense_layer_id = ( - int(ple_dense_layer_id) - if ple_dense_layer_id is not None - else int(layer_idx) - ) - self.prefix = prefix - self.hidden_size = int(config.hidden_size) - self.hc_count = config.hc_count - self.hc_hidden_size = self.hidden_size * self.hc_count - self.conv_kernel_size = int(config.ple_conv_kernel_size) - self.short_conv_dilation = int(config.ngram_size) - self.conv_state_len = (self.conv_kernel_size - 1) * self.short_conv_dilation - self.num_spec_tokens = vllm_config.num_speculative_tokens - self.activation = "silu" - self.ple_embedding: nn.Module = Qwen4ExpNGramEmbedding( - config, - int(config.ple_embed_dim), - self.ple_dense_layer_id, - vllm_config.scheduler_config.max_num_batched_tokens, - f"{prefix}.ple_embedding", - prefix, - data_parallel_rank=vllm_config.parallel_config.data_parallel_rank, - quant_config=quant_config, - params_dtype=model_config.dtype, - use_sequence_parallel=self.use_sequence_parallel, - ) - # Projection weights stay replicated even when their token rows are sharded. - self.kv_proj = MergedColumnParallelLinear( - int(config.ple_embed_dim), - [self.hc_hidden_size, self.hidden_size], - bias=False, - params_dtype=model_config.dtype, - quant_config=quant_config, - prefix=f"{prefix}.kv_proj", - disable_tp=True, - ) - norm_args = ( - self.hc_hidden_size, - config.rms_norm_eps, - self.hidden_size, - model_config.dtype, - ) - self.norm_key = Qwen4ExpPLEGroupedNorm(*norm_args) - self.norm_query = Qwen4ExpPLEGroupedNorm(*norm_args) - self.norm_conv = Qwen4ExpPLEGroupedNorm(*norm_args) - self.conv1d = nn.Conv1d( - self.hc_hidden_size, - self.hc_hidden_size, - self.conv_kernel_size, - groups=self.hc_hidden_size, - padding=self.conv_state_len, - dilation=self.short_conv_dilation, - bias=False, - dtype=model_config.dtype, - ) - nn.init.zeros_(self.conv1d.weight) - self.conv1d.weight._no_reinit = True - self.kv_cache = (torch.tensor([]),) - compilation_config = get_current_vllm_config().compilation_config - if prefix in compilation_config.static_forward_context: - raise ValueError(f"Duplicate layer name: {prefix}") - compilation_config.static_forward_context[prefix] = self - - @property - def mamba_type(self) -> MambaAttentionBackendEnum: - return MambaAttentionBackendEnum.SHORT_CONV - - @property - def is_kv_cache_tp_replicated(self) -> bool: - return True - - def get_attn_backend(self) -> type[PleShortConvAttentionBackend]: - return PleShortConvAttentionBackend - - def get_state_dtype(self) -> tuple[torch.dtype, ...]: - return MambaStateDtypeCalculator.short_conv_state_dtype( - self.model_config.dtype, self.cache_config.mamba_cache_dtype - ) - - def get_state_shape(self) -> Sequence[tuple[int, ...]]: - return MambaStateShapeCalculator.short_conv_state_shape( - tp_world_size=1, - intermediate_size=self.hc_hidden_size, - conv_kernel=self.conv_state_len + 1, - num_spec=self.num_spec_tokens, - ) - - def _short_conv_dilated_dispatch( - self, - inputs: torch.Tensor, - residual: torch.Tensor, - outer_residual: torch.Tensor | None, - metadata: PleShortConvAttentionMetadata, - conv_state: torch.Tensor, - conv_weights: torch.Tensor, - ) -> None: - num_prefills = metadata.num_prefills - num_decodes = metadata.num_decodes - num_decode_tokens = metadata.num_decode_tokens - num_prefill_tokens = metadata.num_prefill_tokens - has_prefill = num_prefills > 0 - has_decode = num_decodes > 0 - has_spec = metadata.spec_sequence_masks is not None - has_non_spec = has_prefill or has_decode - inputs = inputs[: metadata.num_actual_tokens] - residual = residual[: metadata.num_actual_tokens] - if outer_residual is not None: - outer_residual = outer_residual[: metadata.num_actual_tokens] - - spec_token_indices = None - non_spec_token_indices = None - if has_spec and has_non_spec: - assert metadata.spec_token_indx is not None - assert metadata.non_spec_token_indx is not None - spec_token_indices = metadata.spec_token_indx - non_spec_token_indices = metadata.non_spec_token_indx - - if has_spec: - assert metadata.spec_state_indices_tensor is not None - query_start_loc = metadata.spec_query_start_loc - num_accepted_tokens = metadata.num_accepted_tokens - assert query_start_loc is not None - assert num_accepted_tokens is not None - spec_state_indices = metadata.spec_state_indices_tensor[ - : metadata.num_spec_decodes - ] - # Mixed batches stay in their original row order; the kernels map - # logical spec/non-spec rows instead of materializing both groups. - ple_conv( - inputs=inputs, - residual=residual, - conv_state=conv_state, - conv_weights=conv_weights, - state_indices=spec_state_indices, - outer_residual=outer_residual, - mode="spec", - dilation=self.short_conv_dilation, - query_start_loc=query_start_loc, - num_accepted_tokens=num_accepted_tokens, - spec_query_len=metadata.spec_query_len, - token_indices=spec_token_indices, - ) - - if not has_non_spec: - return - - state_indices = metadata.state_indices_tensor - assert state_indices is not None - if has_prefill: - state_indices_d, state_indices_p = torch.split( - state_indices, [num_decodes, num_prefills], dim=0 - ) - if non_spec_token_indices is None: - inputs_d, inputs_p = torch.split( - inputs, [num_decode_tokens, num_prefill_tokens], dim=0 - ) - residual_d, residual_p = torch.split( - residual, [num_decode_tokens, num_prefill_tokens], dim=0 - ) - outer_residual_d: torch.Tensor | None = None - outer_residual_p: torch.Tensor | None = None - if outer_residual is not None: - outer_residual_d, outer_residual_p = torch.split( - outer_residual, - [num_decode_tokens, num_prefill_tokens], - dim=0, - ) - token_indices_d = None - token_indices_p = None - else: - inputs_d = inputs_p = inputs - residual_d = residual_p = residual - outer_residual_d = outer_residual_p = outer_residual - token_indices_d, token_indices_p = torch.split( - non_spec_token_indices, - [num_decode_tokens, num_prefill_tokens], - dim=0, - ) - - if has_decode: - ple_conv( - inputs=inputs_d, - residual=residual_d, - conv_state=conv_state, - conv_weights=conv_weights, - state_indices=state_indices_d, - outer_residual=outer_residual_d, - mode="decode", - dilation=self.short_conv_dilation, - has_initial_states=metadata.has_initial_states_d, - token_indices=token_indices_d, - ) - - query_start_loc = metadata.query_start_loc_p - if query_start_loc is None: - raise ValueError("query_start_loc is required for prefill short-conv") - has_initial_states = metadata.has_initial_states_p - if has_initial_states is None: - raise ValueError( - "has_initial_states_p is required for prefill short-conv" - ) - ple_conv( - inputs=inputs_p, - residual=residual_p, - conv_state=conv_state, - conv_weights=conv_weights, - state_indices=state_indices_p, - outer_residual=outer_residual_p, - mode="prefill", - dilation=self.short_conv_dilation, - query_start_loc=query_start_loc, - has_initial_states=has_initial_states, - token_indices=token_indices_p, - ) - else: - num_decode_rows = ( - non_spec_token_indices.numel() - if non_spec_token_indices is not None - else inputs.size(0) - ) - ple_conv( - inputs=inputs, - residual=residual, - conv_state=conv_state, - conv_weights=conv_weights, - state_indices=state_indices[:num_decode_rows], - outer_residual=outer_residual, - mode="decode", - dilation=self.short_conv_dilation, - has_initial_states=metadata.has_initial_states_d, - token_indices=non_spec_token_indices, - ) - - def _short_conv( - self, - inputs: torch.Tensor, - residual: torch.Tensor, - outer_residual: torch.Tensor | None, - ) -> None: - """Update convolution state, optionally fusing both residual additions.""" - forward_context = get_forward_context() - attn_metadata = forward_context.attn_metadata - # Profiling omits all metadata or this Mamba entry. Short convolution - # is a no-op there, but preserve the outer residual addition. - if attn_metadata is None: - if outer_residual is not None: - residual.add_(outer_residual) - return - - if not isinstance(attn_metadata, dict): - raise RuntimeError( - "PLE short-conv expects per-layer attention metadata dict " - f"during inference, got {type(attn_metadata).__name__}." - ) - - layer_attn_metadata = attn_metadata.get(self.prefix) - if layer_attn_metadata is None: - if outer_residual is not None: - residual.add_(outer_residual) - return - if not isinstance(layer_attn_metadata, PleShortConvAttentionMetadata): - raise TypeError( - "Expected PleShortConvAttentionMetadata for layer " - f"'{self.prefix}', got " - f"{type(layer_attn_metadata).__name__}." - ) - - conv_state = self.kv_cache[0] - # Canonicalize both backend cache layouts to [slot, channel, window]. - if not is_conv_state_dim_first(): - conv_state = conv_state.transpose(-1, -2) - conv_weights = self.conv1d.weight.squeeze(1) - - state_capacity = self.conv_state_len + self.num_spec_tokens - if state_capacity > 0: - if conv_state.size(-1) < state_capacity: - raise RuntimeError( - "PLE short-conv cache is smaller than expected for " - f"dilated convolution: got {conv_state.size(-1)}, " - f"expect at least {state_capacity}." - ) - conv_state = conv_state[..., -state_capacity:] - self._short_conv_dilated_dispatch( - inputs=inputs, - residual=residual, - outer_residual=outer_residual, - metadata=layer_attn_metadata, - conv_state=conv_state, - conv_weights=conv_weights.to(dtype=inputs.dtype), - ) - - def forward( - self, - hidden_states: torch.Tensor, - input_ids: torch.Tensor, - query_start_loc: torch.Tensor, - ngram_context: torch.Tensor, - ) -> torch.Tensor: - input_ids = input_ids.reshape(-1) - if ( - not self.use_sequence_parallel - and input_ids.shape[0] != hidden_states.shape[0] - ): - raise ValueError( - "PLE expects input_ids and hidden_states to have the same " - f"token length, got {input_ids.shape[0]} and " - f"{hidden_states.shape[0]}" - ) - embeddings = self.ple_embedding(input_ids, query_start_loc, ngram_context) - embeddings = self.ple_embedding.ngram_embedding.dequantize( - embeddings, hidden_states.dtype - ) - kv, _ = self.kv_proj(embeddings) - key, value = kv.split(self.kv_proj.output_sizes, dim=-1) - gated_output, conv_input = ple_gate( - key, - value, - hidden_states, - self.norm_key.weight, - self.norm_query.weight, - self.norm_conv.weight, - self.norm_key.eps, - ) - # The short-conv op is a piecewise-graph splitting op: state routing - # reads runtime request metadata. It accumulates the convolution and - # the outer residual into gated_output. - if self.use_sequence_parallel: - # Only convolution needs full tokens. Add residuals on local shards. - conv_input = sp_all_gather(conv_input)[: input_ids.shape[0]] - # Metadata may omit padding tokens or all tokens during profiling. - conv_output = torch.zeros_like(conv_input) - torch.ops.vllm.qwen4_exp_ple_short_conv( - conv_input, conv_output, None, self.prefix - ) - gated_output.add_(sp_shard(conv_output)) - gated_output.add_(hidden_states) - else: - torch.ops.vllm.qwen4_exp_ple_short_conv( - conv_input, - gated_output, - hidden_states, - self.prefix, - ) - return gated_output - - -def qwen4_exp_amd_ple_ngram_ids( - input_ids: torch.Tensor, - query_start_loc: torch.Tensor, - ngram_context: torch.Tensor, - output: torch.Tensor, - layer_name: str, -) -> None: - """Hash the current request layout into n-gram embedding indices. - - The launch sizes its request binary search from the request count, which - is symbolic under Dynamo, so it runs outside Inductor's FX graph. - """ - layer = get_forward_context().no_compile_layers[layer_name] - if not isinstance(layer, Qwen4ExpPLELayer): - raise TypeError(f"{layer_name} is not a Qwen4Exp PLE owner") - embedding = layer.ple_embedding - ple_ngram_ids( - input_ids=input_ids, - query_start_loc=query_start_loc, - ngram_context=ngram_context, - layer_multipliers=embedding.layer_multipliers, - ngram_heads_vocab_sizes=embedding.ngram_heads_vocab_sizes, - ngram_heads_offsets=embedding.ngram_heads_offsets, - eos_token_id=embedding.eos_token_id, - heads_per_ngram=embedding.heads_per_ngram, - output=output, - ) - - -def qwen4_exp_amd_ple_ngram_embedding( - ngram_ids: torch.Tensor, - output: torch.Tensor, - layer_name: str, -) -> None: - """Run the large PLE embedding lookup outside Inductor's FX graph. - - Keeping the embedding weight in ``static_forward_context`` prevents AOT - compile-time autotuning from materializing a synthetic copy of the weight. - """ - layer = get_forward_context().no_compile_layers[layer_name] - if not isinstance(layer, Qwen4ExpPLELayer): - raise TypeError(f"{layer_name} is not a Qwen4Exp PLE owner") - result = layer.ple_embedding.ngram_embedding(ngram_ids).flatten(-2) - output.copy_(result) - - -def qwen4_exp_amd_ple_ngram_embedding_pinned( - ngram_ids: torch.Tensor, - output: torch.Tensor, - layer_name: str, -) -> None: - """Run the pinned PLE UVA lookup outside Inductor's FX graph. - - Same rationale as the device-path escape: keeping the large embedding weight - out of the graph prevents AOT compile-time autotuning from materializing a - synthetic copy of the weight. - """ - layer = get_forward_context().no_compile_layers[layer_name] - if not isinstance(layer, Qwen4ExpPLELayer): - raise TypeError(f"{layer_name} is not a Qwen4Exp PLE owner") - result = layer.ple_embedding.ngram_embedding.sync_lookup(ngram_ids).flatten(-2) - output.copy_(result) - - -def qwen4_exp_ple_short_conv( - inputs: torch.Tensor, - residual: torch.Tensor, - outer_residual: torch.Tensor | None, - layer_name: str, -) -> None: - layer = get_forward_context().no_compile_layers[layer_name] - layer._short_conv(inputs, residual, outer_residual) - - -direct_register_custom_op( - op_name="qwen4_exp_amd_ple_ngram_ids", - op_func=qwen4_exp_amd_ple_ngram_ids, - mutates_args=["output"], -) - - -direct_register_custom_op( - op_name="qwen4_exp_amd_ple_ngram_embedding", - op_func=qwen4_exp_amd_ple_ngram_embedding, - mutates_args=["output"], -) - - -direct_register_custom_op( - op_name="qwen4_exp_amd_ple_ngram_embedding_pinned", - op_func=qwen4_exp_amd_ple_ngram_embedding_pinned, - mutates_args=["output"], -) - - -direct_register_custom_op( - op_name="qwen4_exp_ple_short_conv", - op_func=qwen4_exp_ple_short_conv, - mutates_args=["residual"], -) - - -__all__ = [ - "Qwen4ExpNGramEmbedding", - "Qwen4ExpPLEGroupedNorm", - "Qwen4ExpPLELayer", -] diff --git a/vllm/models/qwen4_exp/amd/qsa.py b/vllm/models/qwen4_exp/amd/qsa.py deleted file mode 100644 index 709fd0b08e..0000000000 --- a/vllm/models/qwen4_exp/amd/qsa.py +++ /dev/null @@ -1,544 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""AMD ROCm QSA owner with Triton kernels.""" - -from __future__ import annotations - -from typing import ClassVar, cast - -import torch -from torch import nn -from transformers import Qwen4ExpTextConfig - -from vllm.config import VllmConfig -from vllm.config.cache import CacheDType -from vllm.distributed import get_tensor_model_parallel_world_size -from vllm.forward_context import get_forward_context -from vllm.model_executor.layers.attention.attention import ( - set_default_quant_scales, -) -from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase -from vllm.model_executor.layers.layernorm import GemmaRMSNorm -from vllm.model_executor.layers.linear import QKVParallelLinear, RowParallelLinear -from vllm.model_executor.layers.quantization import QuantizationConfig -from vllm.model_executor.layers.rotary_embedding import get_rope -from vllm.model_executor.models.qwen3_next import Qwen3NextAttention -from vllm.model_executor.models.qwen3_vl import mrope_positions_factor -from vllm.platforms import current_platform -from vllm.utils.torch_utils import ( - LayerNameType, - _encode_layer_name, - _resolve_layer_name, - canonicalize_singleton_dim_strides, - direct_register_custom_op, - kv_cache_dtype_str_to_dtype, -) -from vllm.v1.attention.backend import ( - AttentionBackend, - AttentionCGSupport, - AttentionType, - MultipleOf, -) -from vllm.v1.attention.backends.fa_utils import is_flash_attn_varlen_func_available -from vllm.v1.attention.backends.flash_attn import ( - FlashAttentionBackend, - FlashAttentionImpl, - FlashAttentionMetadata, - FlashAttentionMetadataBuilder, -) -from vllm.v1.kv_cache_interface import ( - FullAttentionSpec, - KVCacheSpec, - get_kv_quant_mode, -) - -from ..common.qsa_cache import QSAForwardMetadata -from . import model -from .indexer_qsa import QSAIndexer - - -class Qwen4ExpQSAMetadataBuilder(FlashAttentionMetadataBuilder): - """Flash metadata supporting uniform decode and target-verify graphs.""" - - _cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.UNIFORM_BATCH - - -class Qwen4ExpQSAFlashAttentionBackend(FlashAttentionBackend): - """FullAttentionSpec backend used by the merged QSA owner.""" - - supported_dtypes: ClassVar[list[torch.dtype]] = [torch.bfloat16] - supported_kv_cache_dtypes: ClassVar[list[CacheDType]] = ["auto", "bfloat16"] - - @staticmethod - def get_name() -> str: - return "QWEN4_EXP_QSA_TRITON" - - @staticmethod - def get_supported_kernel_block_sizes(kv_cache_spec=None) -> list[int | MultipleOf]: - # QSA consumes manager pages directly and does not use FA4 paged attention. - return [MultipleOf(16)] - - @staticmethod - def get_impl_cls() -> type[Qwen4ExpQSAFlashAttentionImpl]: - return Qwen4ExpQSAFlashAttentionImpl - - @staticmethod - def get_builder_cls() -> type[Qwen4ExpQSAMetadataBuilder]: - return Qwen4ExpQSAMetadataBuilder - - @classmethod - def is_sparse(cls) -> bool: - return True - - @classmethod - def supports_kv_connector(cls) -> bool: - return False - - -class Qwen4ExpQSAFlashAttentionImpl(FlashAttentionImpl): - """Run paged sparse GQA with the QSA Triton kernel.""" - - supports_dcp: bool = False - supports_pcp: bool = False - - def __init__(self, *args, **kwargs) -> None: - super().__init__(*args, **kwargs) - if not is_flash_attn_varlen_func_available(): - raise NotImplementedError("Qwen4Exp QSA requires FlashAttention") - if self.dcp_world_size != 1: - raise NotImplementedError( - "Qwen4Exp QSA does not support decode context parallelism" - ) - if self.kv_cache_dtype not in ("auto", "bfloat16"): - raise NotImplementedError("Qwen4Exp QSA requires a BF16 main KV cache") - self.supports_quant_query_input = False - - def forward_qsa( - self, - layer: torch.nn.Module, - query: torch.Tensor, - key: torch.Tensor | None, - value: torch.Tensor | None, - kv_cache: torch.Tensor, - attn_metadata: FlashAttentionMetadata, - output: torch.Tensor, - token_to_req: torch.Tensor, - output_scale: torch.Tensor | None = None, - output_block_scale: torch.Tensor | None = None, - ) -> torch.Tensor: - del key, value - if output_scale is not None or output_block_scale is not None: - raise NotImplementedError("QSA does not support fused output quantization") - if self.alibi_slopes is not None or self.sinks is not None: - raise NotImplementedError("QSA does not support ALiBi or attention sinks") - if self.sliding_window != (-1, -1): - raise NotImplementedError("QSA does not support sliding-window attention") - - num_tokens = attn_metadata.num_actual_tokens - output.zero_() - if num_tokens == 0: - return output - - topk_buffer = getattr(layer, "topk_indices_buffer", None) - if topk_buffer is None: - raise RuntimeError("QSA owner did not provide its top-k buffer") - logical_indices = topk_buffer[:num_tokens] - token_to_req = token_to_req[:num_tokens] - key_cache, value_cache = kv_cache.transpose(1, 2).split(self.head_size, dim=-1) - key_cache = canonicalize_singleton_dim_strides(key_cache) - value_cache = canonicalize_singleton_dim_strides(value_cache) - if key_cache.dtype != torch.bfloat16 or query.dtype != torch.bfloat16: - raise NotImplementedError("Qwen4Exp QSA requires BF16 Q/K/V") - - from .ops.qsa import qsa_sparse_paged_attention - - qsa_sparse_paged_attention( - query[:num_tokens], - key_cache, - value_cache, - logical_indices, - attn_metadata.block_table, - token_to_req, - output[:num_tokens], - ) - return output - - -class Qwen4ExpQSAAttention(Qwen3NextAttention, AttentionLayerBase): - """Merged Qwen full-attention owner with a QSA index side branch.""" - - supports_dcp = False - - def __init__( - self, - *, - vllm_config: VllmConfig, - config: Qwen4ExpTextConfig, - layer_id: int, - quant_config: QuantizationConfig | None = None, - reduce_results: bool = True, - prefix: str = "", - ) -> None: - nn.Module.__init__(self) - cache_config = vllm_config.cache_config - model_config = vllm_config.model_config - if cache_config is None: - raise ValueError("Qwen4Exp QSA requires a paged KV cache") - if model_config.dtype != torch.bfloat16: - raise NotImplementedError("Qwen4Exp QSA currently requires BF16") - if cache_config.cache_dtype not in ("auto", "bfloat16"): - raise NotImplementedError("Qwen4Exp QSA requires a BF16 main KV cache") - if getattr(quant_config, "kv_cache_scheme", None) is not None: - raise NotImplementedError("Qwen4Exp QSA does not support KV quantization") - parallel_config = vllm_config.parallel_config - if ( - parallel_config.prefill_context_parallel_size > 1 - or parallel_config.decode_context_parallel_size > 1 - ): - raise NotImplementedError( - "Qwen4Exp QSA does not support context parallelism" - ) - if not getattr(config, "is_causal", True): - raise NotImplementedError("Qwen4Exp QSA requires causal decoder attention") - - self.config = config - self.hidden_size = int(config.hidden_size) - tp_size = get_tensor_model_parallel_world_size() - self.total_num_heads = int(config.num_attention_heads) - if self.total_num_heads % tp_size: - raise ValueError("QSA attention heads must be divisible by TP size") - self.num_heads = self.total_num_heads // tp_size - self.total_num_kv_heads = int(config.num_key_value_heads) - if self.total_num_kv_heads >= tp_size: - if self.total_num_kv_heads % tp_size: - raise ValueError("QSA KV heads must be divisible by TP size") - elif tp_size % self.total_num_kv_heads: - raise ValueError("TP size must be divisible by replicated QSA KV heads") - self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size) - self.head_dim = int(config.head_dim or self.hidden_size // self.total_num_heads) - self.q_size = self.num_heads * self.head_dim - self.kv_size = self.num_kv_heads * self.head_dim - self.scaling = self.head_dim**-0.5 - self.dual_chunk_attention_config = getattr( - config, "dual_chunk_attention_config", None - ) - if self.dual_chunk_attention_config is not None: - raise NotImplementedError("Qwen4Exp QSA does not support dual-chunk RoPE") - # Qwen4Exp full-attention checkpoints always pack a sigmoid output - # gate next to Q, even when an inherited config default says otherwise. - self.attn_output_gate = True - - self.qkv_proj = QKVParallelLinear( - self.hidden_size, - self.head_dim, - self.total_num_heads * (1 + self.attn_output_gate), - self.total_num_kv_heads, - bias=False, - quant_config=model.without_modelopt_fp4(quant_config), - prefix=f"{prefix}.qkv_proj", - ) - self.o_proj = RowParallelLinear( - self.total_num_heads * self.head_dim, - self.hidden_size, - bias=False, - reduce_results=reduce_results, - quant_config=quant_config, - prefix=f"{prefix}.o_proj", - ) - self.rotary_emb = get_rope( - head_size=self.head_dim, - max_position=config.max_position_embeddings, - rope_parameters=config.rope_parameters, - mrope_positions_factor=mrope_positions_factor(vllm_config), - ) - self.q_norm = GemmaRMSNorm(self.head_dim, eps=config.rms_norm_eps) - self.k_norm = GemmaRMSNorm(self.head_dim, eps=config.rms_norm_eps) - - mm_config = model_config.multimodal_config - text_only = mm_config is None or mm_config.language_model_only - self.use_fused_qk_norm_rope_gate = ( - self.attn_output_gate - and getattr(self.rotary_emb, "is_neox_style", False) - and current_platform.is_cuda() - and text_only - ) - - self.layer_name = f"{prefix}.attn" - self.attn_type = AttentionType.DECODER - self.kv_cache_dtype = cache_config.cache_dtype - self.kv_cache_torch_dtype = kv_cache_dtype_str_to_dtype( - self.kv_cache_dtype, model_config - ) - if self.kv_cache_torch_dtype != torch.bfloat16: - raise NotImplementedError("Qwen4Exp QSA requires BF16 cache storage") - self.kv_sharing_target_layer_name = None - self.kv_cache = torch.tensor([]) - set_default_quant_scales(self, register_buffer=True) - - self.attn_backend = Qwen4ExpQSAFlashAttentionBackend - self.impl = Qwen4ExpQSAFlashAttentionImpl( - self.num_heads, - self.head_dim, - self.scaling, - self.num_kv_heads, - None, - None, - self.kv_cache_dtype, - None, - AttentionType.DECODER, - None, - ) - self.indexer = QSAIndexer( - vllm_config=vllm_config, - config=config, - layer_id=layer_id, - rotary_emb=self.rotary_emb, - quant_config=quant_config, - prefix=f"{prefix}.indexer", - ) - self.use_fused_qsa_prepare = ( - self.attn_output_gate - and self.indexer.use_fused_pre_indexer - and self.head_dim & (self.head_dim - 1) == 0 - ) - max_tokens = vllm_config.scheduler_config.max_num_batched_tokens - self.register_buffer( - "topk_indices_buffer", - torch.empty( - max_tokens, - self.indexer.output_width, - dtype=torch.int32, - ), - persistent=False, - ) - - static_context = vllm_config.compilation_config.static_forward_context - if self.layer_name in static_context: - raise ValueError(f"Duplicate layer name: {self.layer_name}") - static_context[self.layer_name] = self - - def get_attn_backend(self) -> type[AttentionBackend]: - return self.attn_backend - - def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec: - return FullAttentionSpec( - block_size=vllm_config.cache_config.block_size, - num_kv_heads=self.num_kv_heads, - head_size=self.head_dim, - head_size_v=self.head_dim, - dtype=self.kv_cache_torch_dtype, - kv_quant_mode=get_kv_quant_mode(self.kv_cache_dtype), - ) - - def _run_qsa( - self, - hidden_states: torch.Tensor, - positions: torch.Tensor, - query: torch.Tensor | None, - key: torch.Tensor | None, - value: torch.Tensor | None, - output: torch.Tensor, - qkv: torch.Tensor | None = None, - gate: torch.Tensor | None = None, - ) -> None: - # query/key/value are None when the fused prepare runs inside the - # indexer launch, which then writes the gate into ``gate``. - metadata = get_forward_context().attn_metadata - if isinstance(metadata, list): - metadata = metadata[0] - if not isinstance(metadata, dict): - output.zero_() - if gate is not None: - gate.zero_() - return - main_metadata = cast(FlashAttentionMetadata, metadata[self.layer_name]) - if self.kv_cache.numel() == 0: - raise RuntimeError("QSA main K/V cache is not bound") - - num_tokens = main_metadata.num_actual_tokens - side_metadata = cast( - QSAForwardMetadata, - metadata[self.indexer.raw_key_cache.prefix], - ) - if side_metadata.num_actual_tokens != num_tokens: - raise RuntimeError("QSA main and side metadata token counts disagree") - selected, main_outputs = self.indexer( - hidden_states, - positions, - self.topk_indices_buffer[:num_tokens], - attn=self, - qkv=qkv, - slot_mapping=main_metadata.slot_mapping, - gate_out=gate, - ) - if selected.shape != ( - num_tokens, - self.indexer.output_width, - ): - raise RuntimeError("QSA indexer returned an invalid selection shape") - impl = cast(Qwen4ExpQSAFlashAttentionImpl, self.impl) - if main_outputs is None: - assert key is not None and value is not None - impl.do_kv_cache_update( - self, - key, - value, - self.kv_cache, - main_metadata.slot_mapping, - ) - else: - query, _ = main_outputs - # The gate is applied to every row outside this op; padded rows - # must stay finite so they keep the zero output forward_qsa gives. - assert gate is not None - gate[num_tokens:].zero_() - assert query is not None - impl.forward_qsa( - self, - query, - key, - value, - self.kv_cache, - main_metadata, - output, - token_to_req=side_metadata.token_to_req, - ) - - def forward( - self, - positions: torch.Tensor, - hidden_states: torch.Tensor, - ) -> torch.Tensor: - qkv, _ = self.qkv_proj(hidden_states) - num_tokens = hidden_states.shape[0] - encoded_layer_name = _encode_layer_name(self.layer_name) - if self.use_fused_qsa_prepare: - # Norm/RoPE/gate and the K/V cache write happen inside the op. - attn_output = qkv.new_empty(num_tokens, self.num_heads, self.head_dim) - gate = torch.empty_like(attn_output) - if current_platform.opaque_attention_op(): - torch.ops.vllm.qwen4_exp_qsa_prepare_with_output( - hidden_states, - positions, - qkv, - attn_output, - gate, - encoded_layer_name, - ) - else: - qwen4_exp_qsa_prepare_with_output( - hidden_states, - positions, - qkv, - attn_output, - gate, - encoded_layer_name, - ) - flat_output = attn_output.view(num_tokens, -1) - flat_output = flat_output * torch.sigmoid(gate.view(num_tokens, -1)) - output, _ = self.o_proj(flat_output) - return output - - q, k, v, gate = self._project_qkv_gate(qkv, positions) - query = q.view(num_tokens, self.num_heads, self.head_dim) - key = k.view(num_tokens, self.num_kv_heads, self.head_dim) - value = v.view(num_tokens, self.num_kv_heads, self.head_dim) - attn_output = torch.empty_like(query) - if current_platform.opaque_attention_op(): - torch.ops.vllm.qwen4_exp_qsa_with_output( - hidden_states, - positions, - query, - key, - value, - attn_output, - encoded_layer_name, - ) - else: - qwen4_exp_qsa_with_output( - hidden_states, - positions, - query, - key, - value, - attn_output, - encoded_layer_name, - ) - flat_output = attn_output.view(num_tokens, -1) - if gate is not None: - flat_output = flat_output * torch.sigmoid(gate) - output, _ = self.o_proj(flat_output) - return output - - -def qwen4_exp_qsa_with_output( - hidden_states: torch.Tensor, - positions: torch.Tensor, - query: torch.Tensor, - key: torch.Tensor, - value: torch.Tensor, - output: torch.Tensor, - layer_name: LayerNameType, -) -> None: - """Run the complete QSA state/update/attend transaction.""" - layer_name = _resolve_layer_name(layer_name) - layer = get_forward_context().no_compile_layers[layer_name] - if not isinstance(layer, Qwen4ExpQSAAttention): - raise TypeError(f"{layer_name} is not a Qwen4Exp QSA owner") - layer._run_qsa( - hidden_states, - positions, - query, - key, - value, - output, - ) - - -direct_register_custom_op( - op_name="qwen4_exp_qsa_with_output", - op_func=qwen4_exp_qsa_with_output, - mutates_args=["output"], -) - - -def qwen4_exp_qsa_prepare_with_output( - hidden_states: torch.Tensor, - positions: torch.Tensor, - qkv: torch.Tensor, - output: torch.Tensor, - gate: torch.Tensor, - layer_name: LayerNameType, -) -> None: - """Run the QSA transaction with the main QK-norm/RoPE/gate fused in.""" - layer_name = _resolve_layer_name(layer_name) - layer = get_forward_context().no_compile_layers[layer_name] - if not isinstance(layer, Qwen4ExpQSAAttention): - raise TypeError(f"{layer_name} is not a Qwen4Exp QSA owner") - layer._run_qsa( - hidden_states, - positions, - None, - None, - None, - output, - qkv=qkv, - gate=gate, - ) - - -direct_register_custom_op( - op_name="qwen4_exp_qsa_prepare_with_output", - op_func=qwen4_exp_qsa_prepare_with_output, - mutates_args=["output", "gate"], -) - - -__all__ = [ - "QSAIndexer", - "Qwen4ExpQSAAttention", - "Qwen4ExpQSAFlashAttentionBackend", - "Qwen4ExpQSAFlashAttentionImpl", - "qwen4_exp_qsa_prepare_with_output", - "qwen4_exp_qsa_with_output", -] diff --git a/vllm/parser/AGENTS.md b/vllm/parser/AGENTS.md deleted file mode 100644 index d9e6b79707..0000000000 --- a/vllm/parser/AGENTS.md +++ /dev/null @@ -1,41 +0,0 @@ -# vllm/parser - -Model-output parsing for the serving layer. `Parser` (`abstract_parser.py`) -unifies reasoning extraction and tool-call extraction behind one object; the -serving layer calls `parse` / `parse_delta` and `is_reasoning_end` on it. - -## Layout - -- `abstract_parser.py` — `Parser`, and `DelegatingParser` for wrapping a legacy - `ReasoningParser` + `ToolParser` pair. -- `parser_manager.py` — resolves parser names to classes. -- `engine/` — the declarative parser engine. A `ParserEngineConfig` - (`parser_engine_config.py`) is a set of terminals, transitions and content - events; `parser_engine.py` and `streaming_parser_engine.py` drive it for the - non-streaming and streaming paths; `token_id_scanner.py` pre-lexes - special-token ids and `incremental_lexer.py` lexes text; `adapters.py` - exposes an engine through the legacy reasoning/tool parser interfaces. -- `.py` — one file per model family. Most subclass `ParserEngine` - (`deepseek_v4.py`, `qwen3.py`, ...); a few are `DelegatingParser` wrappers - over legacy parsers (`harmony.py`, `kimi_k3.py`). - -## Guidelines - -- A new parser either subclasses `ParserEngine` with a declarative config, or - subclasses `Parser` and implements `parse` / `parse_delta` directly. Use the - engine when the format is expressible as terminals and transitions, and - subclass an existing config when it is a variant of one (`seed_oss.py`, - `ling3.py`). `DelegatingParser` exists for backward compatibility with the - separate `ReasoningParser` / `ToolParser` classes; do not use it for new - formats. -- Keep a model-specific fix inside that model's `.py` where possible. - Change `engine/` or `abstract_parser.py` only when the behavior is wrong for - every parser, and say so in the PR. -- Text emitted in a state with no `content_events` entry is discarded. Make - that deliberate when adding a state. -- Special tokens reach the engine as token ids, not text. A text terminal that - spans a special token never matches in streaming; list it in - `token_id_terminals`. -- Comment only where the code alone would leave a reader with a question — - a non-obvious transition, a deliberate swallow, a model quirk being worked - around. Keep it minimal and concise. diff --git a/vllm/parser/beam.py b/vllm/parser/beam.py deleted file mode 100644 index 9f99e321f1..0000000000 --- a/vllm/parser/beam.py +++ /dev/null @@ -1,262 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Parser for Beam checkpoints.""" - -from __future__ import annotations - -import json -from typing import TYPE_CHECKING, Any, cast - -from openai.types.responses import ToolChoiceFunction - -from vllm import envs -from vllm.entrypoints.openai.chat_completion.protocol import ( - ChatCompletionNamedToolChoiceParam, - ChatCompletionRequest, -) -from vllm.exceptions import VLLMValidationError -from vllm.logger import init_logger -from vllm.parser.abstract_parser import ( - _xgrammar_supports, - structured_outputs_to_format, -) -from vllm.parser.chat_parsing import ResponseParser -from vllm.parser.chat_parsing.response_templates import ResponseTemplate -from vllm.parser.response_template import ( - CONTENT_FIELD, - THINKING_FIELD, - TOOL_FIELD, - ResponseTemplateParser, - ResponseTemplateReasoningParser, - ResponseTemplateToolParser, - _as_tool_dicts, - _is_named_tool_choice, - _tool_parsing_enabled, -) -from vllm.parser.response_template_grammar import ( - _openers, - answer_rest_structural_tag, - message_start_anchors, - tool_call_structural_tag, -) -from vllm.sampling_params import StructuredOutputsParams -from vllm.tool_parsers.structural_tag_registry import resolve_tool_strictness - -if TYPE_CHECKING: - from vllm.entrypoints.openai.responses.protocol import ResponsesRequest - -logger = init_logger(__name__) - - -def _named_tool(request: ChatCompletionRequest | ResponsesRequest) -> str: - choice = request.tool_choice - if isinstance(choice, ToolChoiceFunction): - return choice.name - return cast(ChatCompletionNamedToolChoiceParam, choice).function.name - - -def _starts_message(template: ResponseTemplate, prefix: str) -> bool: - """Whether output after `prefix` starts the assistant message.""" - ends = [match.end() for match in template.start_anchor_re.finditer(prefix)] - return bool(ends) and ends[-1] == len(prefix) - - -def _open_field(template: ResponseTemplate, prefix: str) -> str | None: - """The field that `prefix` leaves open with its explicit opener, if any.""" - field = None - for event in ResponseParser(template, prefix=prefix).initial_events: - if event["type"] == "region_open" and event["end"] > event["start"]: - field = event["field"] - elif event["type"] in {"region_close", "region_malformed"}: - field = None - return field - - -class BeamReasoningParser(ResponseTemplateReasoningParser): - """Selects `BeamParser` as the reasoning parser.""" - - @classmethod - def unified_parser_cls(cls) -> type[ResponseTemplateParser]: - return BeamParser - - @property - def reasoning_start_str(self) -> str | None: - if literal := super().reasoning_start_str: - return literal - thinking = self.response_template.fields.get(THINKING_FIELD) - if thinking is None: - return None - openers = _openers(thinking, message_start_anchors(self.response_template)) - if not openers: - return None - # Budget tracking needs one marker for both a prompt continuation - # and a complete assistant message in the checkpoint's pattern. - marker = min(openers, key=len) - return marker if all(opener.endswith(marker) for opener in openers) else None - - -class BeamToolParser(ResponseTemplateToolParser): - """Selects `BeamParser` as the tool parser.""" - - @classmethod - def unified_parser_cls(cls) -> type[ResponseTemplateParser]: - return BeamParser - - -class BeamParser(ResponseTemplateParser): - """Parse Beam output with the checkpoint's `response_template` metadata, - and constrain it with a grammar derived from that template.""" - - reasoning_parser_cls = BeamReasoningParser - tool_parser_cls = BeamToolParser - - def __init__(self, *args, **kwargs) -> None: - super().__init__(*args, **kwargs) - chat_template_kwargs = kwargs.get("chat_template_kwargs") or {} - # From the message start, the grammar opens the message with non-empty - # reasoning, unless the request turns thinking off. - self._requires_reasoning = bool( - chat_template_kwargs.get("enable_thinking", True) - ) - - def _constrain_request( - self, - request: ChatCompletionRequest | ResponsesRequest, - ) -> ChatCompletionRequest | ResponsesRequest: - message_start = _starts_message(self.response_template, self._prefix) - tag = self._template_tag(request, message_start=message_start) - if tag is not None: - grammar = StructuredOutputsParams(structural_tag=json.dumps(tag)) - if isinstance(request, ChatCompletionRequest): - request.structured_outputs = grammar - request.response_format = None - else: - request._parser_structured_outputs = grammar - # From the message start, the tag also covers the reasoning. - request._grammar_from_parser = message_start - return request - - def _template_tag( - self, - request: ChatCompletionRequest | ResponsesRequest, - *, - message_start: bool, - ) -> dict[str, Any] | None: - """The template grammar for `request`, or `None` to leave its output - to the request's own constraint, if any.""" - fields = self.response_template.fields - named = _is_named_tool_choice(request) - tools_enabled = ( - self._tool_parser is not None - and bool(request.tools) - and _tool_parsing_enabled( - request, enable_auto_tools=self._enable_auto_tools - ) - ) - forced = tools_enabled and (request.tool_choice == "required" or named) - # The prompt may still list the tools, so only the grammar keeps the - # model from calling them. - tools_forbidden = ( - bool(request.tools) - and request.tool_choice == "none" - and TOOL_FIELD in fields - ) - constrained_tools = None - if tools_enabled and (forced or envs.VLLM_ENFORCE_STRICT_TOOL_CALLING): - assert self._tool_parser is not None - strict_level = self.tool_strict_level - if strict_level is None: - strict_level = self._tool_parser.default_tool_strict_level - constrained_tools = resolve_tool_strictness( - request.tools or [], request.tool_choice, strict_level - ) - output = request.extract_structured_outputs() - if forced and output is not None: - logger.warning_once( - "structured outputs are ignored because tool_choice forces tool call." - ) - output = None - answer = fields.get(CONTENT_FIELD) - if output is not None and ( - output.structural_tag is not None - or answer is None - or answer.open_re is None - or not _xgrammar_supports(output) - ): - if constrained_tools is not None: - logger.warning_once( - "response_template parsing cannot enforce strict tools " - "alongside this structured output." - ) - return None - # Without a tool or structured output constraint, the output still - # follows the template. - always_constrain = envs.VLLM_BEAM_ALWAYS_CONSTRAIN_OUTPUT and not forced - if ( - constrained_tools is None - and output is None - and not tools_forbidden - and not always_constrain - ): - return None - - # Inside an open answer, only the always-on grammar applies, finishing a - # free answer. Inside open reasoning, the grammar needs the reasoning - # gate to defer it to the reasoning's end. - open_field = _open_field(self.response_template, self._prefix) - reasoning_gated = ( - self._reasoning_parser is not None and request.include_reasoning - ) - tag = None - if open_field == CONTENT_FIELD: - if always_constrain and output is None: - tag = answer_rest_structural_tag(fields[CONTENT_FIELD]) - elif open_field != THINKING_FIELD or reasoning_gated: - functions = [] - if tools_enabled: - tools = _as_tool_dicts(constrained_tools or request.tools) - functions = [tool.get("function", tool) for tool in tools] - if named: - name = _named_tool(request) - functions = [f for f in functions if f.get("name") == name] - answer_format = ( - None if output is None else structured_outputs_to_format(output) - ) - try: - tag = tool_call_structural_tag( - fields.get(TOOL_FIELD), - functions, - parallel_tool_calls=( - not named and request.parallel_tool_calls is not False - ), - thinking=fields.get(THINKING_FIELD), - answer=None if forced else answer, - answer_format=( - None if answer_format is None else answer_format.model_dump() - ), - message_start=( - message_start_anchors(self.response_template) - if message_start - else None - ), - requires_reasoning=self._requires_reasoning, - # Beam drifts from schemas the grammar does not enforce, - # and also writes arguments sorted by name. - sorted_arguments=True, - strict_arguments=True, - ) - except ValueError as exc: - raise VLLMValidationError(str(exc), parameter="tools") from exc - if tag is None and forced: - kind = "named" if named else "required" - raise VLLMValidationError( - f"response_template parsing cannot guarantee {kind} tool choice " - "for this request", - parameter="tool_choice", - ) - if tag is None and constrained_tools is not None: - logger.warning_once( - "response_template parsing cannot enforce strict tools for this " - "request." - ) - return tag diff --git a/vllm/parser/cohere_command.py b/vllm/parser/cohere_command.py deleted file mode 100644 index 97a9d3d6f3..0000000000 --- a/vllm/parser/cohere_command.py +++ /dev/null @@ -1,691 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Cohere Command parser. - -Uses ``cohere_melody`` as a stateful parser filter. -""" - -from __future__ import annotations - -import json -from collections.abc import Mapping, Sequence -from typing import TYPE_CHECKING, Any, NamedTuple, TypedDict, TypeGuard - -import regex as re -import xgrammar as xgr - -try: - from cohere_melody import ( - AccumulatedToolCall, - FilterAggregatedResult, - PyFilter, - PyFilterOptions, - ) -except ImportError as e: - raise ImportError( - "The Cohere Command parser requires the `cohere_melody` package. " - "Install it with:\n pip install 'cohere-melody>=0.14.0'" - ) from e - -from vllm.entrypoints.cohere.cohere_chat_message import ( - Citation, - CitationSource, - CohereDeltaMessage, -) -from vllm.entrypoints.generate.base.protocol import ( - AnyResponseFormat, - DeltaFunctionCall, - DeltaMessage, - DeltaToolCall, - FunctionCall, -) -from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest -from vllm.parser.abstract_parser import DelegatingParser -from vllm.reasoning.cohere_command_reasoning_parser import ( - BaseCohereCommandReasoningParser, -) -from vllm.renderers.cohere import POSITION_TO_SOURCE_KEY -from vllm.sampling_params import StructuredOutputsParams, check_json_nesting -from vllm.tool_parsers.cohere_command_tool_parser import BaseCohereCommandToolParser - -if TYPE_CHECKING: - # Declared in melody's stub only; not exported at runtime. - from cohere_melody import FilterCitation, Source - - from vllm.config import ModelConfig - from vllm.entrypoints.openai.responses.protocol import ResponsesRequest - from vllm.tokenizers import TokenizerLike - from vllm.tool_parsers.utils import Tool - -PositionToSource = Mapping[tuple[int, int], CitationSource] - - -class CohereTagRegistry(NamedTuple): - """A single ``structural_tag`` trigger / end pair (``begin`` uses ``trigger``).""" - - trigger: str - end: str - - -class CohereTagStyle(NamedTuple): - """The structural tags style for a given model architecture. - - ``json_tags`` lists every JSON-schema wrapper the model may emit (MOE uses - both response and text delimiters). ``tools`` is the tool-call wrapper. - """ - - json_tags: tuple[CohereTagRegistry, ...] - tools: CohereTagRegistry - - -class CohereNormalizedTool(TypedDict): - """A tool definition normalized to the shape ``collect_tool_schema`` expects. - - ``parameters`` is a JSON Schema object (possibly empty) describing the tool's - call signature. - """ - - name: str - parameters: dict[str, Any] - - -COMMAND_A_TOOLS_TAG = CohereTagRegistry( - trigger="<|START_ACTION|>", - end="<|END_ACTION|>", -) -COMMAND_A_JSON_TAG = CohereTagRegistry( - trigger="<|START_RESPONSE|>", - end="<|END_RESPONSE|>", -) -COMMAND_A_PLUS_JSON_TAG = CohereTagRegistry( - trigger="<|START_TEXT|>", - end="<|END_TEXT|>", -) - -MODEL_TO_TAG_STYLE: dict[str, CohereTagStyle] = { - "Cohere2ForCausalLM": CohereTagStyle( - json_tags=(COMMAND_A_JSON_TAG,), - tools=COMMAND_A_TOOLS_TAG, - ), - "Cohere2VisionForConditionalGeneration": CohereTagStyle( - json_tags=(COMMAND_A_JSON_TAG, COMMAND_A_PLUS_JSON_TAG), - tools=COMMAND_A_TOOLS_TAG, - ), - "Cohere2MoeForCausalLM": CohereTagStyle( - json_tags=(COMMAND_A_JSON_TAG, COMMAND_A_PLUS_JSON_TAG), - tools=COMMAND_A_TOOLS_TAG, - ), -} - - -def collect_tool_schema(tool_schema: list[CohereNormalizedTool]) -> str: - """Build an xgrammar EBNF grammar that matches a JSON array of tool calls. - - The grammar shape is architecture-independent; callers are responsible for - wrapping it in the correct structural tag (see ``CohereTagStyle.tools``). - """ - tool_dictionary: dict[str, str] = {} - for tool in tool_schema: - tool_name = tool["name"] - check_json_nesting(tool["parameters"]) - tool_parameters = dict(tool["parameters"]) - json_schema: dict[str, Any] = { - "type": "object", - "properties": { - "tool_call_id": {"type": "string", "pattern": "^[0-9]+$"}, - "tool_name": {"type": "string", "const": tool_name}, - "parameters": tool_parameters, - }, - } - for defs_key in ("$defs", "definitions"): - if defs_key in tool_parameters: - json_schema[defs_key] = tool_parameters.pop(defs_key) - tool_grammar = str(xgr.Grammar.from_json_schema(json.dumps(json_schema))) - for match in re.findall(r"\b(\w+)\s*::=", tool_grammar): - tool_grammar = re.sub( - rf"\b{re.escape(match)}\b", tool_name + match, tool_grammar - ) - tool_dictionary[tool_name] = f"{tool_name} ::= {tool_name}root\n{tool_grammar}" - # Emitted grammar shape: - # root ::= tools - # tools ::= ws "[" ws tool (ws "," ws tool)* ws "]" ws - # ws ::= (" " | "\t" | "\n")* - # tool ::= | | ... (one alternative per input) - # ::= root (per-tool xgrammar rules) - # root ::= ... (from xgr.Grammar.from_json_schema) - tool_alternatives = "tool ::= " + " | ".join(tool_dictionary.keys()) - tool_rules = "\n ".join(tool_dictionary.values()) - grammar = f"""root ::= tools - tools ::= ws "[" ws tool (ws "," ws tool)* ws "]" ws - ws ::= (" " | "\\t" | "\\n")* - {tool_alternatives} - {tool_rules} - """ - return grammar - - -def _tool_definitions_to_schema_list( - tools: str | list[Any], -) -> list[CohereNormalizedTool]: - """Build the list of ``CohereNormalizedTool`` dicts expected by - ``collect_tool_schema``. - - Accepts: - - JSON string - - list of dicts with top-level ``name`` / ``parameters`` - - list of Chat Completions-style ``{"type": "function", "function": {...}}`` - - list of Pydantic models with ``model_dump()`` - """ - if isinstance(tools, str): - try: - parsed = json.loads(tools) - except json.JSONDecodeError: - return [] - if not isinstance(parsed, list): - return [] - else: - parsed = list(tools) - - out: list[CohereNormalizedTool] = [] - for raw in parsed: - t = raw.model_dump() if hasattr(raw, "model_dump") else raw - if not isinstance(t, dict): - continue - # Unwrap Chat Completions' ``{"type": "function", "function": {...}}`` - # shape; otherwise take the dict as-is. - if t.get("type") == "function" and isinstance(t.get("function"), dict): - t = t["function"] - name = t.get("name") - if not isinstance(name, str): - continue - params = t.get("parameters") - out.append( - CohereNormalizedTool( - name=name, - parameters=params if isinstance(params, dict) else {}, - ) - ) - return out - - -def _has_effective_tools( - tools: str | list[Any] | None, -) -> TypeGuard[str | list[Any]]: - """True when ``tools`` contains at least one tool definition to convert. - - ``ResponsesRequest`` defaults ``tools`` to ``[]``; ``ChatCompletionRequest`` - uses ``None``. Both mean "no tools" here. Strings (e.g. a JSON blob) are - treated as effective only when non-blank. - """ - if tools is None: - return False - if isinstance(tools, str): - return bool(tools.strip()) - return len(tools) > 0 - - -# Builder: produces vLLM response_format in xgrammar's canonical format. -# See xgrammar docs: type "structural_tag" with "format" = triggered_tags -# and tag content type = json_schema | grammar. -def convert_schema_to_structural_tags( - schema: dict | None = None, - tools: str | list[Any] | None = None, - model_architecture: str | None = None, -) -> str | None: - """Returns a response_format string accepted by xgrammar's structural tag format. - Uses the canonical shape: {"type": "structural_tag", "format": {...}} with - format.type "triggered_tags" and tag content type "json_schema" or "grammar". - - Callers that are not on an engine path (e.g. the reasoning parser) must pass - ``model_architecture`` explicitly. - """ - if model_architecture is None or model_architecture not in MODEL_TO_TAG_STYLE: - return None - style = MODEL_TO_TAG_STYLE[model_architecture] - - tags: list[dict] = [] - triggers: list[str] = [] - - def _add_tag(tag: CohereTagRegistry, content: dict) -> None: - tags.append({"begin": tag.trigger, "content": content, "end": tag.end}) - triggers.append(tag.trigger) - - if schema is not None: - # One structural tag per JSON wrapper (e.g. MOE: response + text). - # Same for schema-only and "tools plus JSON mode" (North: schema when - # the model does not call tools). - for jt in style.json_tags: - _add_tag(jt, {"type": "json_schema", "json_schema": schema}) - - if _has_effective_tools(tools): - # ``tools`` may be a JSON string (poseidon / RESPONSE_FORMAT_TOOL_DEFINITIONS) - # or a list (Chat Completions ``request.tools`` as Pydantic models or dicts). - tool_schema_list = _tool_definitions_to_schema_list(tools) - if not tool_schema_list: - raise ValueError( - "No valid tool definitions could be parsed from the request for " - "structural tag conversion." - ) - tool_grammar = collect_tool_schema(tool_schema_list) - _add_tag(style.tools, {"type": "grammar", "grammar": tool_grammar}) - - if not tags: - return None - return json.dumps( - { - "type": "structural_tag", - "format": { - "type": "triggered_tags", - "triggers": triggers, - "tags": tags, - }, - } - ) - - -def _response_format_type( - response_format: AnyResponseFormat | dict | None, -) -> str | None: - if response_format is None: - return None - if isinstance(response_format, dict): - t = response_format.get("type") - return t if isinstance(t, str) else None - return response_format.type - - -def _maybe_parse_json_dict(value: Any) -> dict | None: - """If value is a JSON string, parse to dict; otherwise require dict.""" - if isinstance(value, dict): - return value - if isinstance(value, str): - try: - parsed = json.loads(value) - except (TypeError, json.JSONDecodeError): - return None - return parsed if isinstance(parsed, dict) else None - return None - - -def _unwrap_nested_schema(candidate: Any) -> dict | None: - """Return ``candidate`` as a dict, unwrapping a nested ``schema`` if present. - - Returns ``None`` if ``candidate`` is not (and cannot be parsed into) a dict. - """ - cand = _maybe_parse_json_dict(candidate) - if not isinstance(cand, dict): - return None - nested = cand.get("schema") - return nested if isinstance(nested, dict) else cand - - -def _schema_from_json_schema_field(js_wr: Any) -> dict | None: - """Extract the JSON Schema object from Chat Completions ``json_schema`` payload. - - Accepts: - - ``JsonSchemaResponseFormat`` (Pydantic) with ``schema`` / ``json_schema`` field - - dict in OpenAI shape ``{"name": ..., "schema": {...}}`` - - dict with ``json_schema`` key holding either the schema or a nested wrapper - - dict that is already a JSON Schema document (some clients omit the wrapper) - - JSON strings for any of the above - """ - if js_wr is None: - return None - - parsed_wr = _maybe_parse_json_dict(js_wr) - if parsed_wr is not None: - js_wr = parsed_wr - - if hasattr(js_wr, "model_dump"): - for by_alias in (True, False): - try: - data = js_wr.model_dump(by_alias=by_alias, exclude_none=False) - except TypeError: - data = js_wr.model_dump(by_alias=by_alias) - out = _unwrap_nested_schema(data.get("schema") or data.get("json_schema")) - if out is not None: - return out - inner_attr = getattr(js_wr, "json_schema", None) - return inner_attr if isinstance(inner_attr, dict) else None - - if isinstance(js_wr, dict): - for key in ("schema", "json_schema"): - out = _unwrap_nested_schema(js_wr.get(key)) - if out is not None: - return out - return js_wr - - return None - - -def _schema_dict_from_chat_response_format( - rf: AnyResponseFormat | dict | None, -) -> dict | None: - """JSON schema dict from Chat Completions ``request.response_format`` only.""" - if rf is None: - return None - rf_type = _response_format_type(rf) - if rf_type == "json_object": - return {"type": "object"} - if rf_type != "json_schema": - return None - js_wr = ( - rf.get("json_schema") - if isinstance(rf, dict) - else getattr(rf, "json_schema", None) - ) - return _schema_from_json_schema_field(js_wr) - - -def _schema_dict_from_structured_outputs( - so: StructuredOutputsParams | None, -) -> dict | None: - """Schema dict from ``structured_outputs`` (``json`` / ``json_object``). - - Same unwrapping as ``json_schema``. ``json`` is expected to be ``str`` or - ``dict`` (enforced by ``StructuredOutputsParams`` / request models); other - types raise ``ValueError`` only if a caller bypasses that validation. - """ - if so is None: - return None - if so.json_object: - return {"type": "object"} - raw: Any = so.json - if raw is None: - return None - - if hasattr(raw, "model_dump"): - out = _schema_from_json_schema_field(raw) - if out is None: - raise ValueError( - "structured_outputs.json model has no extractable JSON Schema." - ) - return out - - if isinstance(raw, str): - if not raw.strip(): - raise ValueError("structured_outputs.json cannot be empty.") - check_json_nesting(raw) - try: - raw = json.loads(raw) - except json.JSONDecodeError as e: - raise ValueError("structured_outputs.json must be valid JSON.") from e - if not isinstance(raw, dict): - raise ValueError("structured_outputs.json must decode to a JSON object.") - - if isinstance(raw, Mapping): - body = raw if isinstance(raw, dict) else dict(raw) - return _schema_from_json_schema_field(body) or body - - raise ValueError( - f"structured_outputs.json has unsupported type {type(raw).__name__}." - ) - - -def _melody_sources_to_vllm( - sources: list[Source], position_to_source: PositionToSource | None -) -> list[CitationSource]: - """Resolve melody ``Source`` addresses into :class:`CitationSource` objects. - - Melody reports each source as ``(tool_call_index, tool_result_indices)``. - ``position_to_source`` is built by - :meth:`vllm.entrypoints.cohere.serving.CohereServingChatV2._build_position_to_source` - and forwarded through ``chat_template_kwargs``; each index fans out to one - resolved source, and unresolved positions are skipped. Without a map (a - request that did not come through the Cohere v2 handler) every source is - dropped and the serving layer discards the citation at wire-coercion time. - """ - if not position_to_source: - return [] - out: list[CitationSource] = [] - for source in sources: - for idx in source.tool_result_indices: - info = position_to_source.get((source.tool_call_index, idx)) - if info is not None: - out.append(info) - return out - - -def _melody_citations_to_vllm( - citations: list[FilterCitation], position_to_source: PositionToSource | None -) -> list[Citation] | None: - """Convert melody ``FilterCitation`` objects into :class:`Citation`. - - Citations whose sources all fail to resolve are still returned with - ``sources=[]``; the serving layer drops them so the fail-closed policy - lives in one place. - """ - if not citations: - return None - return [ - Citation( - start=c.start_index, - end=c.end_index, - text=c.text, - sources=_melody_sources_to_vllm(c.sources, position_to_source), - type="THINKING_CONTENT" if c.is_thinking else "TEXT_CONTENT", - ) - for c in citations - ] - - -class CohereCommandParser(DelegatingParser): - def __init__( - self, - tokenizer: TokenizerLike, - tools: list[Tool] | None = None, - *args, - model_config: ModelConfig | None = None, - **kwargs, - ): - super().__init__(tokenizer, tools, *args, model_config=model_config, **kwargs) - self._model_config = model_config - self.start_token_id = tokenizer.convert_tokens_to_ids("<|START_THINKING|>") - self.end_token_id = tokenizer.convert_tokens_to_ids("<|END_THINKING|>") - shim = self._reasoning_parser or self._tool_parser - assert isinstance( - shim, (BaseCohereCommandReasoningParser, BaseCohereCommandToolParser) - ) - options = getattr(PyFilterOptions(), shim.melody_preset)() - self._streaming = PyFilter(options) - self._unary = PyFilter(options) - # Citations from the most recent ``parse`` call, surfaced on - # ``CohereChatMessage`` by ``CohereServingChatV2``. - self.last_unary_citations: list[Citation] | None = None - # Melody may emit a tool-call id before the function name; hold it until - # the first name delta so clients receive both together. - self._pending_tool_ids: dict[int, str] = {} - ctk = kwargs.get("chat_template_kwargs") or {} - raw_map = ctk.get(POSITION_TO_SOURCE_KEY) - self._position_to_source: PositionToSource | None = ( - raw_map if isinstance(raw_map, Mapping) else None - ) - - def adjust_request( - self, request: ChatCompletionRequest | ResponsesRequest - ) -> ChatCompletionRequest | ResponsesRequest: - # keep special tokens to ensure melody can correctly identify boundaries. - request.skip_special_tokens = False - request = self._apply_structural_tags(request) - return super().adjust_request(request) - - def _apply_structural_tags( - self, request: ChatCompletionRequest | ResponsesRequest - ) -> ChatCompletionRequest | ResponsesRequest: - """Fold ``tools`` and any JSON schema into a Cohere ``structural_tag``.""" - so = request.structured_outputs - if so is not None and so.structural_tag: - return request - # Schema: prefer ``response_format`` (OpenAI Chat Completions), then - # ``structured_outputs.json`` / ``json_object`` (vLLM direct). Tools stay - # on ``request.tools``. - rf = ( - request.response_format - if isinstance(request, ChatCompletionRequest) - else None - ) - if rf is not None and _response_format_type(rf) == "structural_tag": - return request - model_architecture = ( - self._model_config.architecture if self._model_config is not None else None - ) - tools = request.tools - # ``response_format`` wins if both it and ``structured_outputs`` supply JSON. - schema = _schema_dict_from_chat_response_format(rf) - if schema is None: - schema = _schema_dict_from_structured_outputs(so) - if schema is None and not _has_effective_tools(tools): - return request - if model_architecture is None: - return request - result = convert_schema_to_structural_tags( - schema=schema, - tools=tools, - model_architecture=model_architecture, - ) - if result is None: - # Unsupported architectures are not in ``MODEL_TO_TAG_STYLE``. - raise ValueError( - "Failed to build structural_tag guided decoding constraints from " - "this request's JSON schema and/or tools. The configured model " - f"architecture ({model_architecture!r}) does not support Cohere " - "command structural tags, or the schema cannot be expressed in " - "that format." - ) - request.structured_outputs = StructuredOutputsParams(structural_tag=result) - # Folded JSON constraints into ``structural_tag``; drop ``response_format`` - # when it was the source so ``to_sampling_params`` does not also set ``json`` / - # ``json_object`` (mutually exclusive in ``StructuredOutputsParams``). - if isinstance(request, ChatCompletionRequest) and rf is not None: - rf_type = _response_format_type(rf) - if rf_type in ("json_schema", "json_object"): - request.response_format = None - return request - - def _apply_structural_tag( - self, request: ChatCompletionRequest | ResponsesRequest - ) -> ChatCompletionRequest | ResponsesRequest: - # Note(arpera): - # No need to go through _apply_structural_tag in abstract_parser.py - # since we have already go through _apply_structural_tags here - # So, override this method to return request as is - return request - - def count_reasoning_tokens(self, token_ids: Sequence[int]) -> int: - count = 0 - depth = 0 - for token_id in token_ids: - if token_id == self.start_token_id: - depth += 1 - continue - if token_id == self.end_token_id: - if depth > 0: - depth -= 1 - continue - if depth > 0: - count += 1 - return count - - def parse( - self, - model_output: str, - request: ChatCompletionRequest | ResponsesRequest, - enable_auto_tools: bool = False, - model_output_token_ids: Sequence[int] = (), - ) -> tuple[str | None, str | None, list[FunctionCall] | None]: - result = self._unary.process_full_text(model_output) - self.last_unary_citations = _melody_citations_to_vllm( - result.citations, self._position_to_source - ) - tool_calls: list[FunctionCall] | None = None - if self._tools_allowed(request): - tool_calls = [ - FunctionCall(id=tc.id, name=tc.name, arguments=tc.arguments) - for tc in result.tool_calls - ] or None - return result.reasoning, result.content, tool_calls - - def parse_delta( - self, - delta_text: str, - delta_token_ids: list[int], - request: ChatCompletionRequest | ResponsesRequest, - prompt_token_ids: list[int] | None = None, - *, - finished: bool, - ) -> DeltaMessage | None: - results = [self._streaming.write_decoded(delta_text)] - if finished: - results.append(self._streaming.flush_partials()) - return self._to_delta(results, request) - - def _tools_allowed(self, request: ChatCompletionRequest | ResponsesRequest) -> bool: - return self._tool_parser is not None and request.tool_choice != "none" - - def _to_delta( - self, - results: list[FilterAggregatedResult], - request: ChatCompletionRequest | ResponsesRequest, - ) -> CohereDeltaMessage | None: - content = self._join(r.content for r in results) - reasoning = ( - self._join(r.reasoning for r in results) - if request.include_reasoning - else None - ) - tool_calls: list[DeltaToolCall] = [] - citations: list[Citation] = [] - for result in results: - if result.tool_calls and self._tools_allowed(request): - tool_calls.extend(self._tool_call_deltas(result.tool_calls)) - citations.extend( - _melody_citations_to_vllm(result.citations, self._position_to_source) - or [] - ) - if not (content or reasoning or tool_calls or citations): - return None - # ``CohereDeltaMessage`` carries citations to the wire via - # ``SerializeAsAny[DeltaMessage]``; it drops the field when unset so the - # shape matches a plain ``DeltaMessage``. - message = CohereDeltaMessage() - if content is not None: - message.content = content - if reasoning is not None: - message.reasoning = reasoning - if tool_calls: - message.tool_calls = tool_calls - if citations: - message.citations = citations - return message - - @staticmethod - def _join(parts: Any) -> str | None: - present = [p for p in parts if p is not None] - return "".join(present) if present else None - - def _tool_call_deltas( - self, melody_tool_calls: list[AccumulatedToolCall] - ) -> list[DeltaToolCall]: - tool_calls: list[DeltaToolCall] = [] - for tc in melody_tool_calls: - name = tc.name or None - arguments = tc.arguments or None - if tc.id: - self._pending_tool_ids[tc.index] = tc.id - # Empty strings are melody streaming placeholders; skip fragments - # that carry neither a name nor argument text. - if name is None and arguments is None: - continue - function = DeltaFunctionCall() - if name is not None: - function.name = name - if arguments is not None: - function.arguments = arguments - delta = DeltaToolCall(index=tc.index, function=function) - if name is not None: - resolved_id = tc.id or self._pending_tool_ids.pop(tc.index, None) - if resolved_id is not None: - delta.id = resolved_id - delta.type = "function" - tool_calls.append(delta) - return tool_calls diff --git a/vllm/parser/deepseek_v32.py b/vllm/parser/deepseek_v32.py deleted file mode 100644 index de6634cf73..0000000000 --- a/vllm/parser/deepseek_v32.py +++ /dev/null @@ -1,144 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""DeepSeek V3.2 parser: DSML tool calls with ``function_calls`` wrapper. - -DeepSeek V3.2 output format:: - - <|DSML|function_calls> - <|DSML|invoke name="func_name"> - <|DSML|parameter name="location" string="true">杭州 - <|DSML|parameter name="count" string="false">5 - - - -This is identical to DeepSeek V4 except for the outer wrapper -(``function_calls`` instead of ``tool_calls``) and the absence of -````/```` reasoning tags. -""" - -from __future__ import annotations - -import functools -from typing import TYPE_CHECKING - -from vllm.parser.deepseek_v4 import ( - DSML_INVOKE_END, - DSML_INVOKE_NAME_END, - DSML_INVOKE_PREFIX, - DSML_PARAM_CLOSE, - DSML_PARAM_START, - _dsml_arg_converter, - _unwrap_wrapper_args, -) -from vllm.parser.engine.events import EventType -from vllm.parser.engine.parser_engine import ParserEngine -from vllm.parser.engine.parser_engine_config import ( - ParserEngineConfig, - ParserState, - Transition, -) - -if TYPE_CHECKING: - from vllm.tokenizers import TokenizerLike - from vllm.tool_parsers.abstract_tool_parser import Tool - -_DSML = "|DSML|" - -DSML_FUNC_START = f"<{_DSML}function_calls>" -DSML_FUNC_END = f"" - - -@functools.cache -def deepseek_v32_config() -> ParserEngineConfig: - return ParserEngineConfig( - name="deepseek_v32", - initial_state=ParserState.CONTENT, - wait_for_reasoning=False, - terminals={ - "TOOL_START": DSML_FUNC_START, - "TOOL_END": DSML_FUNC_END, - "INVOKE_PREFIX": DSML_INVOKE_PREFIX, - "INVOKE_NAME_END": DSML_INVOKE_NAME_END, - "INVOKE_END": DSML_INVOKE_END, - "PARAM_START": DSML_PARAM_START, - "PARAM_CLOSE": DSML_PARAM_CLOSE, - }, - token_id_terminals={ - "TOOL_START": DSML_FUNC_START, - "TOOL_END": DSML_FUNC_END, - }, - transitions={ - (ParserState.CONTENT, "TOOL_START"): Transition( - ParserState.TOOL_PREAMBLE, - (), - ), - (ParserState.TOOL_PREAMBLE, "INVOKE_PREFIX"): Transition( - ParserState.TOOL_NAME, - (EventType.TOOL_CALL_START,), - ), - (ParserState.CONTENT, "INVOKE_PREFIX"): Transition( - ParserState.TOOL_NAME, - (EventType.TOOL_CALL_START,), - ), - (ParserState.TOOL_NAME, "INVOKE_NAME_END"): Transition( - ParserState.TOOL_ARGS, - (), - ), - (ParserState.TOOL_ARGS, "INVOKE_END"): Transition( - ParserState.TOOL_BETWEEN, - (EventType.TOOL_CALL_END,), - ), - (ParserState.TOOL_ARGS, "TOOL_END"): Transition( - ParserState.TOOL_BETWEEN, - (EventType.TOOL_CALL_END,), - ), - # Parallel tool calls - (ParserState.TOOL_BETWEEN, "INVOKE_PREFIX"): Transition( - ParserState.TOOL_NAME, - (EventType.TOOL_CALL_START,), - ), - # A tool call ends the turn: stay in TOOL_BETWEEN so any text - # after the block is dropped. - (ParserState.TOOL_BETWEEN, "TOOL_END"): Transition( - ParserState.TOOL_BETWEEN, - (), - ), - (ParserState.TOOL_BETWEEN, "TOOL_START"): Transition( - ParserState.TOOL_PREAMBLE, - (), - ), - }, - content_events={ - ParserState.CONTENT: EventType.TEXT_CHUNK, - ParserState.TOOL_NAME: EventType.TOOL_NAME, - ParserState.TOOL_ARGS: EventType.ARG_VALUE_CHUNK, - }, - arg_converter=_dsml_arg_converter, - arg_structural_chars=frozenset(">"), - strip_content_whitespace_with_tools=False, - tool_args_json=False, - ) - - -class DeepSeekV32Parser(ParserEngine): - def __init__( - self, - tokenizer: TokenizerLike, - tools: list[Tool] | None = None, - **kwargs, - ) -> None: - kwargs.pop("chat_template_kwargs", None) - super().__init__( - tokenizer, - tools, - parser_engine_config=deepseek_v32_config(), - **kwargs, - ) - self._arg_converter = self._convert_args - - def _convert_args(self, raw_args: str, partial: bool) -> str: - result = _dsml_arg_converter(raw_args, partial) - if not self._tools: - return result - func_name = next((s.name for s in self._tool_slots if s.args == raw_args), None) - return _unwrap_wrapper_args(result, self._tools, func_name) diff --git a/vllm/parser/deepseek_v4.py b/vllm/parser/deepseek_v4.py deleted file mode 100644 index 49ef637365..0000000000 --- a/vllm/parser/deepseek_v4.py +++ /dev/null @@ -1,277 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""DeepSeek V4 parser: ````/```` -reasoning plus DSML tool calls in a single state machine. - -DeepSeek V4 output format:: - - - ...reasoning... - - <|DSML|tool_calls> - <|DSML|invoke name="func_name"> - <|DSML|parameter name="location" string="true">杭州 - <|DSML|parameter name="count" string="false">5 - - -""" - -from __future__ import annotations - -import contextlib -import functools -import json -from typing import TYPE_CHECKING - -import regex as re - -from vllm.parser.engine.events import EventType -from vllm.parser.engine.parser_engine import ParserEngine -from vllm.parser.engine.parser_engine_config import ( - ParserEngineConfig, - ParserState, - Transition, -) -from vllm.tool_parsers.utils import find_tool_properties - -if TYPE_CHECKING: - from vllm.tokenizers import TokenizerLike - from vllm.tool_parsers.abstract_tool_parser import Tool - -_DSML = "|DSML|" - -DSML_THINK_START = "" -DSML_THINK_END = "" -DSML_TOOL_START = f"<{_DSML}tool_calls>" -DSML_TOOL_END = f"" -DSML_INVOKE_PREFIX = f'<{_DSML}invoke name="' -DSML_INVOKE_NAME_END = '">' -DSML_INVOKE_END = f"" -DSML_PARAM_START = f"<{_DSML}parameter" -DSML_PARAM_CLOSE = f"" - -# Spellings variants of ``DSML_TOOL_START`` observed in production. -DSML_TOOL_START_VARIANTS: tuple[str, ...] = ( - f"<{_DSML}toolcalls>", - f"<{_DSML}tool>", -) - - -def _param_patterns( - param_start: str, param_close: str -) -> tuple[re.Pattern, re.Pattern]: - """Complete and trailing-partial parameter regexes for one DSML dialect. - - The ``string`` attribute is optional: the model sometimes omits it, and - dropping such a parameter hands the client a tool call with no arguments. - """ - start, close = re.escape(param_start), re.escape(param_close) - head = rf'{start}\s+name="([^"]+)"(?:\s+string="([^"]*)")?\s*>' - return ( - re.compile(rf"{head}(.*?)(?:{close}|(?={start}\s+name=))", re.DOTALL), - re.compile(rf"{head}(.*)$", re.DOTALL), - ) - - -_PARAM_RE, _PARTIAL_PARAM_RE = _param_patterns(DSML_PARAM_START, DSML_PARAM_CLOSE) - - -def _dsml_arg_converter( - raw_args: str, - partial: bool, - *, - param_re: re.Pattern = _PARAM_RE, - partial_param_re: re.Pattern = _PARTIAL_PARAM_RE, -) -> str: - params: dict[str, object] = {} - - last_end = 0 - # A missing ``string`` attribute is treated like ``string="false"``: try - # JSON, fall back to the literal text. - for m in param_re.finditer(raw_args): - name, is_str, value = m.group(1), m.group(2), m.group(3) - if is_str == "true": - params[name] = value - else: - try: - params[name] = json.loads(value) - except (json.JSONDecodeError, ValueError): - params[name] = value - last_end = m.end() - - if partial: - pm = partial_param_re.search(raw_args, last_end) - if pm: - name, is_str, value = pm.group(1), pm.group(2), pm.group(3) - if is_str == "true": - params[name] = value - else: - with contextlib.suppress(json.JSONDecodeError, ValueError): - params[name] = json.loads(value) - - return json.dumps(params, ensure_ascii=False) - - -def _unwrap_wrapper_args( - args_json: str, - tools: list[Tool] | None, - func_name: str | None, -) -> str: - if not tools or not func_name: - return args_json - try: - args = json.loads(args_json) - except (json.JSONDecodeError, ValueError): - return args_json - if not isinstance(args, dict): - return args_json - properties = find_tool_properties(tools, func_name) - if not properties: - return args_json - allowed = set(properties.keys()) - for wrapper in ("arguments", "input"): - if set(args.keys()) != {wrapper} or wrapper in allowed: - continue - inner = args[wrapper] - if isinstance(inner, str): - try: - inner = json.loads(inner) - except json.JSONDecodeError: - return args_json - if isinstance(inner, dict) and set(inner.keys()).issubset(allowed): - return json.dumps(inner, ensure_ascii=False) - return args_json - - -@functools.cache -def deepseek_v4_config(thinking: bool = False) -> ParserEngineConfig: - return ParserEngineConfig( - name="deepseek_v4", - initial_state=ParserState.REASONING if thinking else ParserState.CONTENT, - wait_for_reasoning=thinking, - terminals={ - "THINK_START": DSML_THINK_START, - "THINK_END": DSML_THINK_END, - "TOOL_START": (DSML_TOOL_START, *DSML_TOOL_START_VARIANTS), - "TOOL_END": DSML_TOOL_END, - "INVOKE_PREFIX": DSML_INVOKE_PREFIX, - "INVOKE_NAME_END": DSML_INVOKE_NAME_END, - "INVOKE_END": DSML_INVOKE_END, - "PARAM_START": DSML_PARAM_START, - "PARAM_CLOSE": DSML_PARAM_CLOSE, - }, - token_id_terminals={ - "THINK_START": DSML_THINK_START, - "THINK_END": DSML_THINK_END, - "TOOL_START": DSML_TOOL_START, - "TOOL_END": DSML_TOOL_END, - }, - transitions={ - (ParserState.CONTENT, "THINK_START"): Transition( - ParserState.REASONING, - (EventType.REASONING_START,), - ), - # Absorb a bare with no prior - (ParserState.CONTENT, "THINK_END"): Transition( - ParserState.CONTENT, - (), - ), - # Absorb a duplicate while already reasoning - (ParserState.REASONING, "THINK_START"): Transition( - ParserState.REASONING, - (), - ), - (ParserState.REASONING, "THINK_END"): Transition( - ParserState.CONTENT, - (EventType.REASONING_END,), - ), - # Tool call beginning while still inside - (ParserState.REASONING, "TOOL_START"): Transition( - ParserState.TOOL_PREAMBLE, - (EventType.REASONING_END,), - ), - (ParserState.CONTENT, "TOOL_START"): Transition( - ParserState.TOOL_PREAMBLE, - (), - ), - (ParserState.TOOL_PREAMBLE, "INVOKE_PREFIX"): Transition( - ParserState.TOOL_NAME, - (EventType.TOOL_CALL_START,), - ), - (ParserState.CONTENT, "INVOKE_PREFIX"): Transition( - ParserState.TOOL_NAME, - (EventType.TOOL_CALL_START,), - ), - (ParserState.TOOL_NAME, "INVOKE_NAME_END"): Transition( - ParserState.TOOL_ARGS, - (), - ), - (ParserState.TOOL_ARGS, "INVOKE_END"): Transition( - ParserState.TOOL_BETWEEN, - (EventType.TOOL_CALL_END,), - ), - (ParserState.TOOL_ARGS, "TOOL_END"): Transition( - ParserState.TOOL_BETWEEN, - (EventType.TOOL_CALL_END,), - ), - # Parallel tool calls - (ParserState.TOOL_BETWEEN, "INVOKE_PREFIX"): Transition( - ParserState.TOOL_NAME, - (EventType.TOOL_CALL_START,), - ), - # A tool call ends the turn: stay in TOOL_BETWEEN so any text - # after the block is dropped. - (ParserState.TOOL_BETWEEN, "TOOL_END"): Transition( - ParserState.TOOL_BETWEEN, - (), - ), - (ParserState.TOOL_BETWEEN, "TOOL_START"): Transition( - ParserState.TOOL_PREAMBLE, - (), - ), - }, - content_events={ - ParserState.CONTENT: EventType.TEXT_CHUNK, - ParserState.REASONING: EventType.REASONING_CHUNK, - ParserState.TOOL_NAME: EventType.TOOL_NAME, - ParserState.TOOL_ARGS: EventType.ARG_VALUE_CHUNK, - }, - arg_converter=_dsml_arg_converter, - arg_structural_chars=frozenset(">"), - strip_content_whitespace_with_tools=False, - tool_args_json=False, - ) - - -class DeepSeekV4Parser(ParserEngine): - parser_config = staticmethod(deepseek_v4_config) - - def __init__( - self, - tokenizer: TokenizerLike, - tools: list[Tool] | None = None, - **kwargs, - ) -> None: - chat_kwargs = kwargs.pop("chat_template_kwargs", None) or {} - thinking = bool( - chat_kwargs.get("thinking") or chat_kwargs.get("enable_thinking") - ) - if "thinking" not in chat_kwargs and "enable_thinking" not in chat_kwargs: - thinking = True - thinking = thinking and chat_kwargs.get("reasoning_effort") != "none" - super().__init__( - tokenizer, - tools, - parser_engine_config=self.parser_config(thinking=thinking), - **kwargs, - ) - self._arg_converter = self._convert_args - - def _convert_args(self, raw_args: str, partial: bool) -> str: - converter = self.parser_engine_config.arg_converter - assert converter is not None - result = converter(raw_args, partial) - if not self._tools: - return result - func_name = next((s.name for s in self._tool_slots if s.args == raw_args), None) - return _unwrap_wrapper_args(result, self._tools, func_name) diff --git a/vllm/parser/deepseek_v41.py b/vllm/parser/deepseek_v41.py deleted file mode 100644 index 8421600510..0000000000 --- a/vllm/parser/deepseek_v41.py +++ /dev/null @@ -1,54 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""DeepSeek V4.1 reasoning and spaced DSML tool calls.""" - -import functools -from dataclasses import replace - -from vllm.parser.deepseek_v4 import ( - DeepSeekV4Parser, - _dsml_arg_converter, - _param_patterns, - deepseek_v4_config, -) -from vllm.parser.engine.parser_engine_config import ParserEngineConfig - -DSML_TOOL_START = "<|DSML| calls>" -DSML_TOOL_END = "" -DSML_INVOKE_PREFIX = '<|DSML| invoke name="' -DSML_INVOKE_END = "" -DSML_PARAM_START = "<|DSML| parameter" -DSML_PARAM_CLOSE = "" - -_PARAM_RE, _PARTIAL_PARAM_RE = _param_patterns(DSML_PARAM_START, DSML_PARAM_CLOSE) - - -@functools.cache -def deepseek_v41_config(thinking: bool = False) -> ParserEngineConfig: - config = deepseek_v4_config(thinking=thinking) - terminal_overrides = { - "TOOL_START": DSML_TOOL_START, - "TOOL_END": DSML_TOOL_END, - "INVOKE_PREFIX": DSML_INVOKE_PREFIX, - "INVOKE_END": DSML_INVOKE_END, - "PARAM_START": DSML_PARAM_START, - "PARAM_CLOSE": DSML_PARAM_CLOSE, - } - return replace( - config, - name="deepseek_v41", - terminals={**config.terminals, **terminal_overrides}, - token_id_terminals={ - key: terminal_overrides.get(key, value) - for key, value in config.token_id_terminals.items() - }, - arg_converter=functools.partial( - _dsml_arg_converter, - param_re=_PARAM_RE, - partial_param_re=_PARTIAL_PARAM_RE, - ), - ) - - -class DeepSeekV41Parser(DeepSeekV4Parser): - parser_config = staticmethod(deepseek_v41_config) diff --git a/vllm/parser/engine/registered_adapters.py b/vllm/parser/engine/registered_adapters.py index 6d610ce3b5..302344efe3 100644 --- a/vllm/parser/engine/registered_adapters.py +++ b/vllm/parser/engine/registered_adapters.py @@ -7,112 +7,10 @@ names so that :class:`ReasoningParserManager` and :class:`ToolParserManager` can load them lazily. """ -from vllm.parser.deepseek_v4 import DeepSeekV4Parser -from vllm.parser.deepseek_v32 import DeepSeekV32Parser -from vllm.parser.deepseek_v41 import DeepSeekV41Parser from vllm.parser.engine.adapters import make_adapters -from vllm.parser.gemma4 import Gemma4Parser -from vllm.parser.glm47_moe import Glm47MoeParser -from vllm.parser.granite import GraniteParser -from vllm.parser.granite_thinking import GraniteThinkingParser -from vllm.parser.inkling import InklingParser -from vllm.parser.kimi_k2 import KimiK2Parser -from vllm.parser.mimo import MiMoParser -from vllm.parser.minimax_m2 import MinimaxM2Parser -from vllm.parser.minimax_m3 import MinimaxM3Parser -from vllm.parser.mistral import MistralParser -from vllm.parser.nemotron_v3 import NemotronV3Parser -from vllm.parser.plamo3 import Plamo3Parser from vllm.parser.qwen3 import Qwen3Parser -from vllm.parser.seed_oss import SeedOssParser -from vllm.parser.step3p5 import Step3p5Parser - -( - DeepSeekV32ParserReasoningAdapter, - DeepSeekV32ParserToolAdapter, -) = make_adapters(DeepSeekV32Parser) - -( - DeepSeekV4ParserReasoningAdapter, - DeepSeekV4ParserToolAdapter, -) = make_adapters(DeepSeekV4Parser) - -( - DeepSeekV41ParserReasoningAdapter, - DeepSeekV41ParserToolAdapter, -) = make_adapters(DeepSeekV41Parser) - -( - MinimaxM2ParserReasoningAdapter, - MinimaxM2ParserToolAdapter, -) = make_adapters(MinimaxM2Parser) - -( - MinimaxM3ParserReasoningAdapter, - MinimaxM3ParserToolAdapter, -) = make_adapters(MinimaxM3Parser) - -( - Gemma4ParserReasoningAdapter, - Gemma4ParserToolAdapter, -) = make_adapters(Gemma4Parser) - -( - GraniteParserReasoningAdapter, - GraniteParserToolAdapter, -) = make_adapters(GraniteParser) - -( - GraniteThinkingParserReasoningAdapter, - GraniteThinkingParserToolAdapter, -) = make_adapters(GraniteThinkingParser) - -( - NemotronV3ParserReasoningAdapter, - NemotronV3ParserToolAdapter, -) = make_adapters(NemotronV3Parser) ( Qwen3ParserReasoningAdapter, Qwen3ParserToolAdapter, ) = make_adapters(Qwen3Parser) - -( - MiMoParserReasoningAdapter, - MiMoParserToolAdapter, -) = make_adapters(MiMoParser) - -( - SeedOssParserReasoningAdapter, - SeedOssParserToolAdapter, -) = make_adapters(SeedOssParser) - -( - Step3p5ParserReasoningAdapter, - Step3p5ParserToolAdapter, -) = make_adapters(Step3p5Parser) - -( - Glm47MoeParserReasoningAdapter, - Glm47MoeParserToolAdapter, -) = make_adapters(Glm47MoeParser) - -( - KimiK2ParserReasoningAdapter, - KimiK2ParserToolAdapter, -) = make_adapters(KimiK2Parser) - -( - InklingParserReasoningAdapter, - InklingParserToolAdapter, -) = make_adapters(InklingParser) - -( - MistralParserReasoningAdapter, - MistralParserToolAdapter, -) = make_adapters(MistralParser) - -( - Plamo3ParserReasoningAdapter, - Plamo3ParserToolAdapter, -) = make_adapters(Plamo3Parser) diff --git a/vllm/parser/gemma4.py b/vllm/parser/gemma4.py deleted file mode 100644 index 738ba58295..0000000000 --- a/vllm/parser/gemma4.py +++ /dev/null @@ -1,636 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Gemma4 parser. - -Handles channel-based reasoning plus custom tool call format in a single -state machine:: - - <|channel>thought - ...reasoning... - <|tool_call>call:func_name{key:<|"|>value<|"|>,num:42} -""" - -from __future__ import annotations - -import functools -import json -from collections.abc import Sequence -from typing import TYPE_CHECKING - -from vllm.entrypoints.generate.base.protocol import DeltaMessage -from vllm.logger import init_logger -from vllm.parser.engine.events import EventType, SemanticEvent -from vllm.parser.engine.parser_engine import ParserEngine -from vllm.parser.engine.parser_engine_config import ( - ArgValueExtender, - ParserEngineConfig, - ParserState, - Transition, -) - -if TYPE_CHECKING: - from vllm.entrypoints.openai.chat_completion.protocol import ( - ChatCompletionRequest, - ) - from vllm.entrypoints.openai.responses.protocol import ResponsesRequest - from vllm.tokenizers import TokenizerLike - from vllm.tool_parsers.abstract_tool_parser import Tool - -CHANNEL_START = "<|channel>" -CHANNEL_END = "" -TOOL_CALL_START = "<|tool_call>" -TOOL_CALL_END = "" -STRING_DELIM = '<|"|>' -_DELIM_LEN = len(STRING_DELIM) - -logger = init_logger(__name__) - - -# --------------------------------------------------------------------------- -# Gemma4 argument parser -# --------------------------------------------------------------------------- - -_PARTIAL_DELIM_SUFFIXES = tuple( - STRING_DELIM[:k] for k in range(len(STRING_DELIM), 0, -1) -) - - -def _strip_partial_delim(value: str) -> str: - """Strip a trailing partial ``STRING_DELIM`` prefix from *value*. - - Prevents partial delimiters from leaking into the streamed JSON diff. - """ - for suffix in _PARTIAL_DELIM_SUFFIXES: - if value.endswith(suffix): - return value[: -len(suffix)] - return value - - -def _parse_gemma4_args(args_str: str, *, partial: bool = False) -> dict: - return _parse_gemma4_fields(args_str, partial=partial)[0] - - -def _parse_gemma4_fields( - args_str: str, *, partial: bool = False -) -> tuple[dict, str | None]: - """Parse Gemma4's custom key:value format into a Python dict. - - Format examples:: - - location:<|"|>Tokyo<|"|> - location:<|"|>San Francisco<|"|>,unit:<|"|>celsius<|"|> - count:42,flag:true - nested:{inner_key:<|"|>val<|"|>} - items:[<|"|>a<|"|>,<|"|>b<|"|>] - - Args: - args_str: The raw Gemma4 argument string. - partial: When True (streaming), bare values at end of string are - omitted because they may be incomplete and type-unstable - (e.g. partial boolean parsed as bare string). - - Returns a dict ready for ``json.dumps()`` and the key of the trailing - top-level ``STRING_DELIM`` value left unterminated, if any. - - """ - if not args_str or not args_str.strip(): - return {}, None - - result: dict = {} - open_key: str | None = None - i = 0 - n = len(args_str) - - while i < n: - while i < n and args_str[i] in (" ", ",", "\n", "\t"): - i += 1 - if i >= n: - break - - key_start = i - while i < n and args_str[i] != ":": - i += 1 - if i >= n: - break - key = args_str[key_start:i].strip() - if key.startswith(STRING_DELIM) and key.endswith(STRING_DELIM): - key = key[_DELIM_LEN:-_DELIM_LEN] - i += 1 - - if i >= n: - if not partial: - result[key] = "" - break - - while i < n and args_str[i] in (" ", "\n", "\t"): - i += 1 - if i >= n: - if not partial: - result[key] = "" - break - - if args_str.startswith(STRING_DELIM, i): - i += _DELIM_LEN - val_start = i - end_pos = args_str.find(STRING_DELIM, i) - if end_pos == -1: - # Unterminated string — take rest, strip partial delimiter. - value = args_str[val_start:] - if partial: - value = _strip_partial_delim(value) - result[key] = value - open_key = key - break - result[key] = args_str[val_start:end_pos] - i = end_pos + _DELIM_LEN - - elif args_str[i] == "{": - depth = 1 - obj_start = i + 1 - i += 1 - while i < n and depth > 0: - if args_str.startswith(STRING_DELIM, i): - # Skip over string contents to avoid counting { inside strings - i += _DELIM_LEN - next_delim = args_str.find(STRING_DELIM, i) - i = n if next_delim == -1 else next_delim + _DELIM_LEN - continue - if args_str[i] == "{": - depth += 1 - elif args_str[i] == "}": - depth -= 1 - i += 1 - if depth > 0: - # Incomplete nested object — use i (not i-1) to avoid - # dropping the last char, and recurse as partial. - result[key] = _parse_gemma4_args(args_str[obj_start:i], partial=True) - else: - result[key] = _parse_gemma4_args(args_str[obj_start : i - 1]) - - elif args_str[i] == "[": - depth = 1 - arr_start = i + 1 - i += 1 - while i < n and depth > 0: - if args_str.startswith(STRING_DELIM, i): - i += _DELIM_LEN - next_delim = args_str.find(STRING_DELIM, i) - i = n if next_delim == -1 else next_delim + _DELIM_LEN - continue - if args_str[i] == "[": - depth += 1 - elif args_str[i] == "]": - depth -= 1 - i += 1 - if depth > 0: - result[key] = _parse_gemma4_array(args_str[arr_start:i], partial=True) - else: - result[key] = _parse_gemma4_array(args_str[arr_start : i - 1]) - - else: - val_start = i - while i < n and args_str[i] not in (",", "}", "]"): - i += 1 - if partial and i >= n: - # Value may be incomplete (e.g. partial boolean) — - # withhold to avoid type instability during streaming. - break - if i == val_start: - logger.warning( - "Gemma4 args parser made no progress at position %d; " - "aborting on malformed input.", - i, - ) - break - raw_val = args_str[val_start:i].strip() - if partial and raw_val.endswith("."): - # Digits may still arrive (e.g. "108." -> "108.2"); - # withhold to avoid corrupting the streaming diff. - break - result[key] = raw_val - - return result, open_key - - -def _parse_gemma4_array(arr_str: str, *, partial: bool = False) -> list: - items: list = [] - i = 0 - n = len(arr_str) - - while i < n: - while i < n and arr_str[i] in (" ", ",", "\n", "\t"): - i += 1 - if i >= n: - break - - if arr_str.startswith(STRING_DELIM, i): - i += _DELIM_LEN - end_pos = arr_str.find(STRING_DELIM, i) - if end_pos == -1: - items.append(arr_str[i:]) - break - items.append(arr_str[i:end_pos]) - i = end_pos + _DELIM_LEN - - elif arr_str[i] == "{": - depth = 1 - obj_start = i + 1 - i += 1 - while i < n and depth > 0: - if arr_str.startswith(STRING_DELIM, i): - i += _DELIM_LEN - nd = arr_str.find(STRING_DELIM, i) - i = nd + _DELIM_LEN if nd != -1 else n - continue - if arr_str[i] == "{": - depth += 1 - elif arr_str[i] == "}": - depth -= 1 - i += 1 - if depth > 0: - items.append(_parse_gemma4_args(arr_str[obj_start:i], partial=True)) - else: - items.append(_parse_gemma4_args(arr_str[obj_start : i - 1])) - - elif arr_str[i] == "[": - depth = 1 - sub_start = i + 1 - i += 1 - while i < n and depth > 0: - if arr_str.startswith(STRING_DELIM, i): - i += _DELIM_LEN - nd = arr_str.find(STRING_DELIM, i) - i = nd + _DELIM_LEN if nd != -1 else n - continue - if arr_str[i] == "[": - depth += 1 - elif arr_str[i] == "]": - depth -= 1 - i += 1 - if depth > 0: - items.append(_parse_gemma4_array(arr_str[sub_start:i], partial=True)) - else: - items.append(_parse_gemma4_array(arr_str[sub_start : i - 1])) - - else: - val_start = i - while i < n and arr_str[i] not in (",", "]"): - i += 1 - if partial and i >= n: - break - if i == val_start: - logger.warning( - "Gemma4 array parser made no progress at position %d; " - "aborting on malformed input.", - i, - ) - break - raw_val = arr_str[val_start:i].strip() - if partial and raw_val.endswith("."): - break - items.append(raw_val) - - return items - - -def _drop_call_closer(text: str, paren_excess: bool) -> str: - """Drop a trailing ``}`` or, with more ``)`` than ``(``, a trailing ``)``. - - The closer of the call body is dropped before parsing, so a value that - itself ends in one loses that character until more text arrives. - """ - if text.endswith("}") or (text.endswith(")") and paren_excess): - return text[:-1] - return text - - -def _gemma4_arg_converter(raw_args: str, partial: bool) -> str: - """Convert Gemma4 custom arg format to a JSON string.""" - text = raw_args.strip() - text = _drop_call_closer(text, text.count("(") < text.count(")")) - parsed = _parse_gemma4_args(text, partial=partial) - return json.dumps(parsed, ensure_ascii=False) - - -def _gemma4_arg_value_extender(raw_args: str) -> ArgValueExtender | None: - """Extend the open top-level ``STRING_DELIM`` value at the end of - *raw_args*, if it is the last value in the converted JSON. - - The extender streams exactly what ``_gemma4_arg_converter`` would, so it - holds back the same tail: trailing whitespace, a dropped closer and a - partial ``STRING_DELIM``. ``STRING_DELIM`` is a special token but not a - lexer terminal, so unlike a tag it may reach the extender split across - deltas; the held tail doubles as the window that detects the close. - """ - text = raw_args.strip() - opens, closes = text.count("("), text.count(")") - body = _drop_call_closer(text, opens < closes) - params, open_key = _parse_gemma4_fields(body, partial=True) - if open_key is None or next(reversed(params)) != open_key: - return None - - held_len = ( - len(raw_args) - - len(raw_args.rstrip()) - + len(text) - - len(_strip_partial_delim(body)) - ) - held = raw_args[len(raw_args) - held_len :] if held_len else "" - - def extend(delta: str) -> str | None: - nonlocal held, opens, closes - text = held + delta - if STRING_DELIM in text: - return None - opens += delta.count("(") - closes += delta.count(")") - visible = _strip_partial_delim(_drop_call_closer(text.rstrip(), opens < closes)) - held = text[len(visible) :] - return json.dumps(visible, ensure_ascii=False)[1:-1] - - return extend - - -@functools.cache -def gemma4_config(thinking: bool = False) -> ParserEngineConfig: - return ParserEngineConfig( - name="gemma4", - initial_state=ParserState.CONTENT, - # Decides a silent tail: the E-series HF templates write no marker - # with thinking off, and no template does after a tool response. - wait_for_reasoning=thinking, - turn_boundary_tokens=frozenset({"<|turn>", "<|tool_response>"}), - terminals={ - "THINK_START": CHANNEL_START, - "THINK_END": CHANNEL_END, - "TOOL_START": TOOL_CALL_START, - "TOOL_END": TOOL_CALL_END, - "CALL_PREFIX": "call:", - "COLON": ":", - "OPEN_BRACE": "{", - "OPEN_PAREN": "(", - }, - token_id_terminals={ - "THINK_START": CHANNEL_START, - "THINK_END": CHANNEL_END, - "TOOL_START": TOOL_CALL_START, - "TOOL_END": TOOL_CALL_END, - }, - transitions={ - # -- Reasoning transitions -- - (ParserState.CONTENT, "THINK_START"): Transition( - ParserState.REASONING, - (EventType.REASONING_START,), - ), - # No-op: if we pre-initialised the engine to REASONING from the - # prompt (see ``adjust_initial_state_from_prompt``) but the model - # still emits its own ``<|channel>`` opener, swallow it instead - # of leaking it as TEXT_CHUNK. - (ParserState.REASONING, "THINK_START"): Transition( - ParserState.REASONING, - (), - ), - (ParserState.REASONING, "THINK_END"): Transition( - ParserState.CONTENT, - (EventType.REASONING_END,), - ), - # Tool call directly from reasoning (no explicit ) - (ParserState.REASONING, "TOOL_START"): Transition( - ParserState.TOOL_PREAMBLE, - (EventType.REASONING_END, EventType.TOOL_CALL_START), - ), - # -- Tool call transitions -- - (ParserState.CONTENT, "TOOL_START"): Transition( - ParserState.TOOL_PREAMBLE, - (EventType.REASONING_END, EventType.TOOL_CALL_START), - ), - (ParserState.TOOL_PREAMBLE, "TOOL_END"): Transition( - ParserState.CONTENT, - (EventType.TOOL_CALL_END,), - ), - (ParserState.TOOL_PREAMBLE, "CALL_PREFIX"): Transition( - ParserState.TOOL_NAME, - (), - ), - # Bare opener: some checkpoints emit "<|tool_call>:name{...}" - # without the "call" prefix. Match the colon directly so the - # tool call is not silently dropped. - (ParserState.TOOL_PREAMBLE, "COLON"): Transition( - ParserState.TOOL_NAME, - (), - ), - (ParserState.TOOL_NAME, "OPEN_BRACE"): Transition( - ParserState.TOOL_ARGS, - (), - ), - (ParserState.TOOL_NAME, "OPEN_PAREN"): Transition( - ParserState.TOOL_ARGS, - (), - ), - (ParserState.TOOL_NAME, "TOOL_END"): Transition( - ParserState.CONTENT, - (EventType.TOOL_CALL_END,), - ), - (ParserState.TOOL_ARGS, "TOOL_END"): Transition( - ParserState.CONTENT, - (EventType.TOOL_CALL_END,), - ), - # Back-to-back tool calls - (ParserState.CONTENT, "TOOL_END"): Transition( - ParserState.CONTENT, - (), - ), - # Absorb a bare that arrives after we already - # returned to CONTENT; prevents leaking it as TEXT_CHUNK. - (ParserState.CONTENT, "THINK_END"): Transition( - ParserState.CONTENT, - (), - ), - }, - content_events={ - ParserState.CONTENT: EventType.TEXT_CHUNK, - ParserState.REASONING: EventType.REASONING_CHUNK, - ParserState.TOOL_NAME: EventType.TOOL_NAME, - ParserState.TOOL_ARGS: EventType.ARG_VALUE_CHUNK, - }, - arg_converter=_gemma4_arg_converter, - arg_value_extender=_gemma4_arg_value_extender, - tool_args_json=False, - arg_structural_chars=frozenset(",:{}[]<"), - preserve_tokens=frozenset({STRING_DELIM}), - ) - - -_GEMMA4_THOUGHT_PREFIX = "thought\n" -_GEMMA4_THOUGHT_TOKEN = "thought" - - -class Gemma4Parser(ParserEngine): - """Gemma4 parser: ``<|channel>`` reasoning + ``<|tool_call>`` - tool calls in a single engine. - - - Strips the ``thought\\n`` prefix from reasoning content - - Sets ``skip_special_tokens=False`` so boundary tokens are visible - - Detects ``<|tool_call>`` token as implicit reasoning end - """ - - def __init__( - self, - tokenizer: TokenizerLike, - tools: list[Tool] | None = None, - **kwargs, - ) -> None: - chat_kwargs = kwargs.get("chat_template_kwargs", {}) or {} - thinking = chat_kwargs.get("enable_thinking", False) - kwargs.setdefault("parser_engine_config", gemma4_config(thinking=thinking)) - super().__init__(tokenizer, tools, **kwargs) - vocab = self.vocab - self._tool_call_token_id: int | None = vocab.get("<|tool_call>") - self._new_turn_token_id: int | None = vocab.get("<|turn>") - self._tool_response_token_id: int | None = vocab.get("<|tool_response>") - self._reasoning_text: str = "" - self._prefix_stripped: bool = False - self._is_first_feed: bool = True - - def _reset(self, initial_state=None) -> None: - super()._reset(initial_state=initial_state) - self._reasoning_text = "" - self._prefix_stripped = False - self._is_first_feed = True - - def _preprocess_feed( - self, - delta_text: str, - delta_token_ids: Sequence[int], - ) -> tuple[str, Sequence[int]]: - if not self._is_first_feed: - return delta_text, delta_token_ids - self._is_first_feed = False - - if ( - not delta_text - or self._engine.state != ParserState.CONTENT - or self._reasoning_start_token_id is None - or self._reasoning_end_token_id is None - ): - return delta_text, delta_token_ids - - if CHANNEL_START in delta_text: - return delta_text, delta_token_ids - - needs_injection = ( - CHANNEL_END in delta_text - or delta_text.startswith(_GEMMA4_THOUGHT_PREFIX) - or delta_text == _GEMMA4_THOUGHT_TOKEN - ) - if not needs_injection: - return delta_text, delta_token_ids - - delta_text = CHANNEL_START + delta_text - if delta_token_ids: - delta_token_ids = [self._reasoning_start_token_id, *delta_token_ids] - - return delta_text, delta_token_ids - - def _prompt_ends_in_open_reasoning(self, prompt_token_ids: Sequence[int]) -> bool: - """Whether the prompt tail is inside an open ``<|channel>`` block. - - Scans backwards: a ``<|channel>`` start token seen before any - closing or turn-boundary token means the block is still open. - """ - start_id = self._reasoning_start_token_id - if start_id is None: - return False - boundary_ids = { - tid - for tid in ( - self._reasoning_end_token_id, - self._tool_call_token_id, - self._new_turn_token_id, - self._tool_response_token_id, - ) - if tid is not None - } - for tid in reversed(prompt_token_ids): - if tid == start_id: - return True - if tid in boundary_ids: - return False - return False - - def adjust_initial_state_from_prompt(self, prompt_token_ids: Sequence[int]) -> None: - """Pre-initialise the engine to ``REASONING`` when the prompt ends - inside an open ``<|channel>`` block. - - This covers the post-tool-response continuation case where the chat - template leaves the prompt ending with ``<|channel>thought\n`` - (issue #45834). A prompt that merely starts a new model turn must - not pre-initialise reasoning: the model may answer directly without - emitting any channel markers, and the non-streaming path classifies - such output as content (issue #48217). When the model does open its - own ``<|channel>``, the ``(CONTENT, THINK_START)`` transition - handles it, and the ``thought\n`` prefix in the first reasoning - chunk is stripped by ``_events_to_delta`` as in the default flow. - """ - if not self._prompt_ends_in_open_reasoning(prompt_token_ids): - return - self._engine.reset(initial_state=ParserState.REASONING) - # Prevent a later default ``initialize_streaming()`` (e.g. from - # ``ParserEngineReasoningAdapter.extract_reasoning_streaming``) from - # clobbering this with ``CONTENT``. - self._streaming_initialized = True - - def _events_to_delta( - self, - events: list[SemanticEvent], - finished: bool = False, - ) -> DeltaMessage | None: - delta = super()._events_to_delta(events, finished=finished) - if delta is None or delta.reasoning is None: - return delta - - if self._prefix_stripped: - return delta - self._reasoning_text += delta.reasoning - - if self._reasoning_text.startswith(_GEMMA4_THOUGHT_PREFIX): - prefix_len = len(_GEMMA4_THOUGHT_PREFIX) - prev_reasoning_len = len(self._reasoning_text) - len(delta.reasoning) - if prev_reasoning_len >= prefix_len: - self._prefix_stripped = True - return delta - chars_of_prefix_in_delta = prefix_len - prev_reasoning_len - stripped = delta.reasoning[chars_of_prefix_in_delta:] - if stripped: - self._prefix_stripped = True - delta.reasoning = stripped - return delta - if len(self._reasoning_text) >= prefix_len: - self._prefix_stripped = True - delta.reasoning = None - if delta.content is not None or delta.tool_calls: - return delta - return None - return None - - if _GEMMA4_THOUGHT_PREFIX.startswith(self._reasoning_text): - if finished: - self._prefix_stripped = True - return None - - self._prefix_stripped = True - delta.reasoning = self._reasoning_text - return delta - - def extract_reasoning( - self, - model_output: str, - request: ChatCompletionRequest | ResponsesRequest, - ) -> tuple[str | None, str | None]: - reasoning, content = super().extract_reasoning(model_output, request) - if reasoning: - if reasoning.startswith(_GEMMA4_THOUGHT_PREFIX): - reasoning = reasoning[len(_GEMMA4_THOUGHT_PREFIX) :] - elif reasoning == _GEMMA4_THOUGHT_PREFIX.rstrip(): - reasoning = None - return reasoning or None, content diff --git a/vllm/parser/glm47_moe.py b/vllm/parser/glm47_moe.py deleted file mode 100644 index 44377e62b7..0000000000 --- a/vllm/parser/glm47_moe.py +++ /dev/null @@ -1,286 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""GLM-4.7 parser for reasoning and tool calls. - -GLM-4.7 uses XML-like tool calls:: - - func_namekeyvalue - -The function name can be followed directly by the first ```` tag, -and tool calls may have no arguments. -""" - -from __future__ import annotations - -import functools -import json -from typing import TYPE_CHECKING - -import regex as re - -from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest -from vllm.entrypoints.openai.responses.protocol import ResponsesRequest -from vllm.logger import init_logger -from vllm.parser.engine.events import EventType -from vllm.parser.engine.parser_engine import ParserEngine -from vllm.parser.engine.parser_engine_config import ( - ArgValueExtender, - ParserEngineConfig, - ParserState, - Transition, -) - -logger = init_logger(__name__) - -if TYPE_CHECKING: - from vllm.tokenizers import TokenizerLike - from vllm.tool_parsers.abstract_tool_parser import Tool - -THINK_START = "" -THINK_END = "" -TOOL_CALL_START = "" -TOOL_CALL_END = "" -ARG_KEY_START = "" -ARG_KEY_END = "" -ARG_VALUE_START = "" -ARG_VALUE_END = "" - -# Special tokens that delimit conversation turns in a rendered GLM prompt. -# Reasoning markers belonging to earlier turns must not be mistaken for the -# state of the turn currently being generated. -GLM_TURN_BOUNDARIES = frozenset( - ("<|system|>", "<|user|>", "<|assistant|>", "<|observation|>") -) - -_ARG_RE = re.compile( - r"(?P.*?)\s*" - r"(?P.*?)", - re.DOTALL, -) -_PARTIAL_ARG_RE = re.compile( - r"(?P.*?)\s*" - r"(?P.*)$", - re.DOTALL, -) - - -def _glm47_parse_args( - raw_args: str, partial: bool -) -> tuple[dict[str, object], str | None]: - """Return the arguments and the key of the open trailing value, if any.""" - params: dict[str, object] = {} - - for match in _ARG_RE.finditer(raw_args): - params[match.group("key").strip()] = match.group("value") - - open_key = None - if partial: - remaining = _ARG_RE.sub("", raw_args) - match = _PARTIAL_ARG_RE.search(remaining) - if match: - key = match.group("key").strip() - if key: - params[key] = match.group("value") - open_key = key - - return params, open_key - - -def _glm47_arg_converter(raw_args: str, partial: bool) -> str: - return json.dumps(_glm47_parse_args(raw_args, partial)[0], ensure_ascii=False) - - -def _glm47_arg_value_extender(raw_args: str) -> ArgValueExtender | None: - # The open value runs to the end of raw_args and is the last JSON value, - # so a delta appends its JSON-escaped text unless it closes the value. - params, open_key = _glm47_parse_args(raw_args, partial=True) - if open_key is None or next(reversed(params)) != open_key: - return None - - keep = len(ARG_VALUE_END) - 1 - tail = raw_args[-keep:] - - def extend(delta: str) -> str | None: - nonlocal tail - text = tail + delta - if ARG_VALUE_END in text: - return None - tail = text[-keep:] - return json.dumps(delta, ensure_ascii=False)[1:-1] - - return extend - - -def _glm53_always_thinks(tokenizer: TokenizerLike) -> bool: - # GLM-5.3 keeps the GLM-4.x prompt and tool-call format but replaces the - # enable_thinking toggle with an always-on "Reasoning Effort:" header, so - # its generation prompt always opens and the model output carries - # no opening tag. - template = getattr(tokenizer, "chat_template", None) - return ( - isinstance(template, str) - and "[gMASK]" in template - and "Reasoning Effort:" in template - and "enable_thinking" not in template - and "" in template - and "" in template - and "" in template - ) - - -@functools.cache -def glm47_moe_config(thinking: bool = True) -> ParserEngineConfig: - arg_tag_transitions = { - (ParserState.TOOL_ARGS, terminal): Transition( - ParserState.TOOL_ARGS, - (EventType.ARG_VALUE_CHUNK,), - ) - for terminal in ( - "ARG_KEY_START", - "ARG_KEY_END", - "ARG_VALUE_START", - "ARG_VALUE_END", - ) - } - - reasoning_terminals = ( - { - "THINK_START": THINK_START, - "THINK_END": THINK_END, - } - if thinking - else {} - ) - reasoning_token_id_terminals = ( - { - "THINK_START": THINK_START, - "THINK_END": THINK_END, - } - if thinking - else {} - ) - reasoning_transitions = ( - { - (ParserState.CONTENT, "THINK_START"): Transition( - ParserState.REASONING, - (EventType.REASONING_START,), - ), - (ParserState.REASONING, "THINK_END"): Transition( - ParserState.CONTENT, - (EventType.REASONING_END,), - ), - (ParserState.CONTENT, "THINK_END"): Transition( - ParserState.CONTENT, - (), - ), - } - if thinking - else {} - ) - - return ParserEngineConfig( - name="glm47_moe", - initial_state=ParserState.REASONING if thinking else ParserState.CONTENT, - terminals={ - **reasoning_terminals, - "TOOL_START": TOOL_CALL_START, - "TOOL_END": TOOL_CALL_END, - "ARG_KEY_START": ARG_KEY_START, - "ARG_KEY_END": ARG_KEY_END, - "ARG_VALUE_START": ARG_VALUE_START, - "ARG_VALUE_END": ARG_VALUE_END, - }, - token_id_terminals={ - **reasoning_token_id_terminals, - "TOOL_START": TOOL_CALL_START, - "TOOL_END": TOOL_CALL_END, - }, - transitions={ - **reasoning_transitions, - (ParserState.REASONING, "THINK_START"): Transition( - ParserState.REASONING, - (), - ), - (ParserState.REASONING, "TOOL_START"): Transition( - ParserState.TOOL_NAME, - (EventType.REASONING_END, EventType.TOOL_CALL_START), - ), - (ParserState.CONTENT, "TOOL_START"): Transition( - ParserState.TOOL_NAME, - (EventType.TOOL_CALL_START,), - ), - (ParserState.TOOL_NAME, "ARG_KEY_START"): Transition( - ParserState.TOOL_ARGS, - (EventType.ARG_VALUE_CHUNK,), - ), - (ParserState.TOOL_NAME, "TOOL_END"): Transition( - ParserState.CONTENT, - (EventType.TOOL_CALL_END,), - ), - (ParserState.TOOL_ARGS, "TOOL_END"): Transition( - ParserState.CONTENT, - (EventType.TOOL_CALL_END,), - ), - **arg_tag_transitions, - }, - turn_boundary_tokens=GLM_TURN_BOUNDARIES, - arg_converter=_glm47_arg_converter, - arg_value_extender=_glm47_arg_value_extender, - stream_arg_deltas=True, - tool_args_json=False, - validate_tool_names=True, - ) - - -class Glm47MoeParser(ParserEngine): - """GLM-4.7 parser backed by the declarative parser engine.""" - - def __init__( - self, - tokenizer: TokenizerLike, - tools: list[Tool] | None = None, - **kwargs, - ) -> None: - chat_kwargs = kwargs.get("chat_template_kwargs", {}) or {} - thinking = chat_kwargs.get("thinking", None) - enable_thinking = chat_kwargs.get("enable_thinking", None) - if (thinking is False or enable_thinking is False) and _glm53_always_thinks( - tokenizer - ): - logger.warning_once( - "Ignoring enable_thinking/thinking: the GLM-5.3 chat template " - "has no thinking switch, so reasoning is always on and " - "disabling extraction would leak it into the content." - ) - thinking = None - enable_thinking = None - self.thinking_enabled = ( - True - if thinking is None and enable_thinking is None - else bool(thinking) or bool(enable_thinking) - ) - kwargs.setdefault( - "parser_engine_config", - glm47_moe_config(thinking=self.thinking_enabled), - ) - super().__init__(tokenizer, tools, **kwargs) - - def _emit_name_delta(self, idx: int, deltas, name: str | None) -> None: - if name is not None: - name = name.strip() - super()._emit_name_delta(idx, deltas, name) - - def _handle_tool_end(self, event, deltas) -> None: - idx = event.tool_index - if 0 <= idx < len(self._tool_slots): - self._tool_slots[idx].name = self._tool_slots[idx].name.strip() - super()._handle_tool_end(event, deltas) - - def extract_reasoning( - self, - model_output: str, - request: ChatCompletionRequest | ResponsesRequest, - ) -> tuple[str | None, str | None]: - if not self.thinking_enabled: - return None, model_output - return super().extract_reasoning(model_output, request) diff --git a/vllm/parser/granite.py b/vllm/parser/granite.py deleted file mode 100644 index afa2021225..0000000000 --- a/vllm/parser/granite.py +++ /dev/null @@ -1,88 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Granite parser for JSON-array tool calls. - -Granite 3.0/3.1 emit tool calls as a JSON array following a marker:: - - <|tool_call|> [{"name": "get_weather", "arguments": {"city": "SF"}}] # 3.0 - [{"name": "get_weather", "arguments": {"city": "SF"}}] # 3.1 - -``<|tool_call|>`` is a single special token; ```` is plain text. -The array may hold several calls, has no closing marker (the ``]`` ends it), -and any surrounding prose is content. The engine's ``tool_call_body_array`` -mode splits the array into one call per element and extracts each name via the -name-from-args path; ``_granite_arg_converter`` carves the ``arguments`` object -out of each ``{"name":..,"arguments":{..}}`` element. Granite has no reasoning. -""" - -from __future__ import annotations - -import functools -from typing import TYPE_CHECKING - -from vllm.parser.engine.parser_engine import ParserEngine -from vllm.parser.engine.parser_engine_config import ( - ParserEngineConfig, - ParserState, - Transition, -) -from vllm.parser.utils import extract_json_member_object - -if TYPE_CHECKING: - from vllm.tokenizers import TokenizerLike - from vllm.tool_parsers.abstract_tool_parser import Tool - -TOOL_TOKEN = "<|tool_call|>" # granite 3.0 (single special token) -TOOL_STRING = "" # granite 3.1 (plain text) - - -def _granite_arg_converter(raw_args: str, partial: bool) -> str: - """Carve the ``arguments`` object out of a ``{"name":..,"arguments":{..}}`` - array element (see :func:`extract_json_member_object`).""" - span = extract_json_member_object(raw_args, "arguments") - if span is None: - return "" if partial else "{}" - return span - - -@functools.cache -def granite_config() -> ParserEngineConfig: - return ParserEngineConfig( - name="granite", - initial_state=ParserState.CONTENT, - terminals={ - "TOOL_TOKEN": TOOL_TOKEN, - "TOOL_STRING": TOOL_STRING, - }, - token_id_terminals={ - "TOOL_TOKEN": TOOL_TOKEN, - }, - transitions={ - # The marker only switches into the tool region; the array splitter - # emits TOOL_CALL_START per element (empty events here). - (ParserState.CONTENT, "TOOL_TOKEN"): Transition( - ParserState.TOOL_ARGS, - (), - ), - (ParserState.CONTENT, "TOOL_STRING"): Transition( - ParserState.TOOL_ARGS, - (), - ), - }, - arg_converter=_granite_arg_converter, - tool_call_body_array=True, - stream_arg_deltas=True, - ) - - -class GraniteParser(ParserEngine): - CONFIG_NAME = "granite" - - def __init__( - self, - tokenizer: TokenizerLike, - tools: list[Tool] | None = None, - **kwargs, - ) -> None: - kwargs.setdefault("parser_engine_config", granite_config()) - super().__init__(tokenizer, tools, **kwargs) diff --git a/vllm/parser/granite_thinking.py b/vllm/parser/granite_thinking.py deleted file mode 100644 index 3ab3c8c0af..0000000000 --- a/vllm/parser/granite_thinking.py +++ /dev/null @@ -1,131 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Granite 4.2 thinking parser. - -Granite 4.2 uses the same tool call and reasoning format as Nemotron V3 -(````/```` + ```` XML). This config reuses -:class:`NemotronV3Parser` with two Granite-specific behaviors: - -1. Strip leading newlines from content — the Granite chat template writes - ``\\n\\n``, so the first content character is always ``\\n``. -2. Inherit the Nemotron V3 reasoning-to-content swap for - ``enable_thinking=False`` / ``force_nonempty_content=True``. -""" - -from __future__ import annotations - -import dataclasses -import functools -from typing import TYPE_CHECKING - -from vllm.parser.nemotron_v3 import NemotronV3Parser, nemotron_v3_config -from vllm.parser.qwen3 import Qwen3Parser - -if TYPE_CHECKING: - from vllm.entrypoints.generate.base.protocol import DeltaMessage - from vllm.entrypoints.openai.chat_completion.protocol import ( - ChatCompletionRequest, - ) - from vllm.entrypoints.openai.responses.protocol import ResponsesRequest - from vllm.parser.engine.parser_engine import SemanticEvent - from vllm.parser.engine.parser_engine_config import ParserEngineConfig - from vllm.tokenizers import TokenizerLike - from vllm.tool_parsers.abstract_tool_parser import Tool - - -@functools.cache -def granite_thinking_config(thinking: bool = True) -> ParserEngineConfig: - return dataclasses.replace( - nemotron_v3_config(thinking=thinking), - name="granite_thinking_parser", - ) - - -class GraniteThinkingParser(NemotronV3Parser): - """Granite 4.2 parser: same format as Nemotron V3, with - leading-newline stripping on content after ````. - - The ``reasoning_ended`` property is overridden to delay the - reasoning-to-content transition reported to - :class:`~vllm.parser.abstract_parser.DelegatingParser` until the - template's leading newline has been consumed. Without this delay, - ``DelegatingParser.parse_delta`` passes newline tokens through a - raw pass-through path that bypasses the engine's - ``_events_to_delta`` (and therefore our stripping logic). - """ - - def __init__( - self, - tokenizer: TokenizerLike, - tools: list[Tool] | None = None, - **kwargs, - ) -> None: - chat_kwargs = kwargs.get("chat_template_kwargs", {}) or {} - thinking = chat_kwargs.get("enable_thinking", True) - kwargs.pop("parser_engine_config", None) - Qwen3Parser.__init__( - self, - tokenizer, - tools, - parser_engine_config=granite_thinking_config(thinking=thinking), - **kwargs, - ) - self._streamed_reasoning: list[str] = [] - self._content_started = False - - @property - def reasoning_ended(self) -> bool: - # Internal engine logic uses _reasoning_ended directly; this - # property is only read by DelegatingParser (via the adapter's - # has_engine_confirmed_reasoning_end) to decide when to stop - # routing deltas through extract_reasoning_streaming. - if not self._reasoning_ended: - return False - return self._content_started - - def _reset(self, initial_state=None) -> None: - super()._reset(initial_state=initial_state) - self._content_started = False - - def _events_to_delta( - self, - events: list[SemanticEvent], - finished: bool = False, - ) -> DeltaMessage | None: - delta = super()._events_to_delta(events, finished=finished) - if ( - delta is not None - and delta.content is not None - and not self._content_started - ): - stripped = delta.content.lstrip("\n") - if not stripped: - delta.content = None - if delta.reasoning is None and not delta.tool_calls: - return None - else: - self._content_started = True - delta.content = stripped - return delta - - def extract_reasoning( - self, - model_output: str, - request: ChatCompletionRequest | ResponsesRequest, - ) -> tuple[str | None, str | None]: - reasoning, content = Qwen3Parser.extract_reasoning(self, model_output, request) - - # Track whether content was truly absent (None) vs present - # but whitespace-only — the swap should only fire for absent. - content_was_absent = content is None - - if content is not None: - content = content.lstrip("\n") or None - - # Only swap when content was truly absent from model output - # (truncated reasoning or max_tokens cut at ), not - # when it was present but stripped to empty (newlines-only). - if self._should_force_content(request) and content_was_absent: - reasoning, content = content, reasoning - - return reasoning, content diff --git a/vllm/parser/inkling.py b/vllm/parser/inkling.py deleted file mode 100644 index 46be6f02b6..0000000000 --- a/vllm/parser/inkling.py +++ /dev/null @@ -1,332 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Inkling parser: typed content blocks parsed by a single state machine. - -Inkling output format (every marker is a dedicated special token):: - - <|message_model|><|content_thinking|>...reasoning...<|end_message|> - <|message_model|><|content_text|>...visible text...<|end_message|> - <|message_model|><|content_invoke_tool_json|> - {"name":"get_weather","args":{"city":"SF"}}<|end_message|> - -Blocks are self-describing and may repeat in any order; sampling may -also end a block with the standalone ``<|content_model_end_sampling|>`` -token. The tool-call payload is a single JSON object whose ``name`` is -extracted by the engine's name-from-args path and whose ``args`` object -is carved out of the wrapper by :func:`_inkling_arg_converter`. - -Note the terminal *labels*: ``THINK_START``/``THINK_END`` are what the -engine keys its reasoning plumbing on (``is_reasoning_end``, -``count_reasoning_tokens``, initial-state seeding), so ``<|end_message|>`` -is labelled ``THINK_END`` here even though it ends every block kind — -the transition table, not the label, carries the semantics. -""" - -from __future__ import annotations - -import functools -import json -from collections.abc import Sequence -from typing import TYPE_CHECKING - -from vllm.entrypoints.generate.base.protocol import ExtractedToolCallInformation -from vllm.parser.engine.events import EventType -from vllm.parser.engine.parser_engine import ParserEngine -from vllm.parser.engine.parser_engine_config import ( - ParserEngineConfig, - ParserState, - Transition, -) -from vllm.parser.utils import extract_json_member_object - -if TYPE_CHECKING: - from vllm.tokenizers import TokenizerLike - from vllm.tool_parsers.abstract_tool_parser import Tool - -MESSAGE_MODEL = "<|message_model|>" -CONTENT_TEXT = "<|content_text|>" -CONTENT_THINKING = "<|content_thinking|>" -CONTENT_INVOKE_TOOL_JSON = "<|content_invoke_tool_json|>" -CONTENT_INVOKE_TOOL_TEXT = "<|content_invoke_tool_text|>" -CONTENT_TOOL_ERROR = "<|content_tool_error|>" -CONTENT_MODEL_END_SAMPLING = "<|content_model_end_sampling|>" -END_MESSAGE = "<|end_message|>" - -INKLING_SPECIAL_TOKENS = ( - MESSAGE_MODEL, - CONTENT_TEXT, - CONTENT_THINKING, - CONTENT_INVOKE_TOOL_JSON, - CONTENT_INVOKE_TOOL_TEXT, - CONTENT_TOOL_ERROR, - CONTENT_MODEL_END_SAMPLING, - END_MESSAGE, -) - - -def _inkling_arg_converter(raw_args: str, partial: bool) -> str: - """Carve the ``args`` object out of the tool-call JSON wrapper. - - Why a converter at all: the engine's ``tool_args_json`` machinery - treats the *entire* TOOL_ARGS text as the tool arguments, but Inkling's - payload is the ``{"name":...,"args":{...}}`` wrapper — without a - converter, ``_compute_arg_delta`` streams the wrapper verbatim into - the OpenAI ``arguments`` field (``converter is None -> raw delta``, - unconditionally; ``stream_arg_deltas=False`` does not stop it). - :func:`extract_json_member_object` returns a prefix-stable verbatim - substring, which the engine's argument-delta diffing requires. - """ - span = extract_json_member_object(raw_args, "args") - if span is None: - # No args value yet (streaming) or none at all (treat as empty). - return "" if partial else "{}" - return span - - -@functools.cache -def inkling_config() -> ParserEngineConfig: - terminals = { - "MSG_MODEL": MESSAGE_MODEL, - "TEXT_START": CONTENT_TEXT, - "THINK_START": CONTENT_THINKING, - "THINK_END": END_MESSAGE, - "END_SAMPLING": CONTENT_MODEL_END_SAMPLING, - "TOOL_START": CONTENT_INVOKE_TOOL_JSON, - "TOOL_TEXT": CONTENT_INVOKE_TOOL_TEXT, - "TOOL_ERROR": CONTENT_TOOL_ERROR, - } - transitions: dict[tuple[ParserState, str], Transition] = { - # ── Between blocks / inside a text block ────────────────────── - (ParserState.CONTENT, "MSG_MODEL"): Transition( - ParserState.MESSAGE_HEADER, - (), - ), - # Opening a block that renders as visible content proves no reasoning - # is still open. Confirming that here is what lets DelegatingParser - # hand the following blocks to the tool pass: the reasoning pass only - # leaves its reasoning phase on an explicit reasoning-end event, and a - # response with no thinking block never emits one otherwise. - (ParserState.CONTENT, "TEXT_START"): Transition( - ParserState.CONTENT, - (EventType.REASONING_END,), - ), - (ParserState.CONTENT, "THINK_START"): Transition( - ParserState.REASONING, - (EventType.REASONING_START,), - ), - # A tool block confirms the same boundary, and is the one opener that - # can start a turn with no visible block ahead of it. - (ParserState.CONTENT, "TOOL_START"): Transition( - ParserState.TOOL_ARGS, - (EventType.REASONING_END, EventType.TOOL_CALL_START), - ), - # Raw / error tool blocks render as visible text. - (ParserState.CONTENT, "TOOL_TEXT"): Transition( - ParserState.CONTENT, - (EventType.REASONING_END,), - ), - (ParserState.CONTENT, "TOOL_ERROR"): Transition( - ParserState.CONTENT, - (EventType.REASONING_END,), - ), - # The optional function name between the model-role and content-kind - # markers is metadata, not visible assistant content. - (ParserState.MESSAGE_HEADER, "MSG_MODEL"): Transition( - ParserState.MESSAGE_HEADER, - (), - ), - (ParserState.MESSAGE_HEADER, "TEXT_START"): Transition( - ParserState.CONTENT, - (EventType.REASONING_END,), - ), - (ParserState.MESSAGE_HEADER, "THINK_START"): Transition( - ParserState.REASONING, - (EventType.REASONING_START,), - ), - (ParserState.MESSAGE_HEADER, "TOOL_START"): Transition( - ParserState.TOOL_ARGS, - (EventType.REASONING_END, EventType.TOOL_CALL_START), - ), - (ParserState.MESSAGE_HEADER, "TOOL_TEXT"): Transition( - ParserState.CONTENT, - (EventType.REASONING_END,), - ), - (ParserState.MESSAGE_HEADER, "TOOL_ERROR"): Transition( - ParserState.CONTENT, - (EventType.REASONING_END,), - ), - # ── Inside a thinking block ─────────────────────────────────── - (ParserState.REASONING, "THINK_START"): Transition( - ParserState.REASONING, - (), - ), - # Defensive: tool call opening while a thinking block is still - # unclosed. - (ParserState.REASONING, "TOOL_START"): Transition( - ParserState.TOOL_ARGS, - (EventType.REASONING_END, EventType.TOOL_CALL_START), - ), - } - # A closed tool block returns to CONTENT (Inkling has no section wrapper; - # blocks of any kind may follow), which also keeps the block-kind - # and role tokens out of the engine's tool-terminal set so the - # skip_tool_parsing reasoning pass still classifies reasoning. - # In a reasoning block, THINK_END defers REASONING_END to the next block - # opener so the function-name header is not emitted as content. - for end in ("THINK_END", "END_SAMPLING"): - transitions[(ParserState.CONTENT, end)] = Transition( - ParserState.CONTENT, - (), - ) - transitions[(ParserState.REASONING, end)] = Transition( - ParserState.CONTENT, - () if end == "THINK_END" else (EventType.REASONING_END,), - ) - transitions[(ParserState.TOOL_ARGS, end)] = Transition( - ParserState.CONTENT, - (EventType.TOOL_CALL_END,), - ) - transitions[(ParserState.MESSAGE_HEADER, end)] = Transition( - ParserState.CONTENT, - (EventType.TEXT_CHUNK,), - ) - - return ParserEngineConfig( - name="inkling", - # Normal generation continues after a prompt-prefilled - # `<|message_model|>`. Non-streaming parsing receives only the generated - # suffix, so begin in the corresponding message-header state as well. - initial_state=ParserState.MESSAGE_HEADER, - wait_for_reasoning=True, - terminals=terminals, - # Inkling content-kind markers are the grammar. When the engine is - # used through DelegatingParser, the reasoning pass can hand the tool - # pass reconstructed text whose token-id slice no longer contains the - # content-kind marker that starts the tool block, so keep Inkling on - # the text grammar instead of token-id-only terminal matching. - token_id_terminals={}, - transitions=transitions, - arg_converter=_inkling_arg_converter, - stream_arg_deltas=True, - tool_args_json=True, - strip_trailing_reasoning_whitespace=True, - drop_whitespace_only_content_before_tools=True, - strip_content_whitespace_with_tools=False, - validate_tool_names=False, - ) - - -class InklingParser(ParserEngine): - CONFIG_NAME = "inkling" - - def __init__( - self, - tokenizer: TokenizerLike, - tools: list[Tool] | None = None, - **kwargs, - ) -> None: - kwargs.setdefault("parser_engine_config", inkling_config()) - super().__init__(tokenizer, tools, **kwargs) - - def adjust_initial_state_from_prompt(self, prompt_token_ids: Sequence[int]) -> None: - """Seed the initial parsing state from the prompt tail. - - Mirrors the Rust parser's ``initialize()``: scanning the prompt - backwards, the last relevant special token decides whether generation - continues inside a thinking block, a text block, a model message - header, or between blocks. - """ - vocab = self.vocab - thinking_id = vocab.get(CONTENT_THINKING) - text_id = vocab.get(CONTENT_TEXT) - model_id = vocab.get(MESSAGE_MODEL) - special_ids = {vocab[text] for text in INKLING_SPECIAL_TOKENS if text in vocab} - for token_id in reversed(prompt_token_ids): - if token_id == thinking_id: - self._engine.reset(initial_state=ParserState.REASONING) - self._streaming_initialized = True - return - if token_id == text_id: - self._engine.reset(initial_state=ParserState.CONTENT) - self._streaming_initialized = True - return - if token_id == model_id: - self._engine.reset(initial_state=ParserState.MESSAGE_HEADER) - self._streaming_initialized = True - return - if token_id in special_ids: - break - - def is_reasoning_end(self, input_ids: list[int]) -> bool: - vocab = self.vocab - thinking_id = vocab.get(CONTENT_THINKING) - text_id = vocab.get(CONTENT_TEXT) - model_id = vocab.get(MESSAGE_MODEL) - end_sampling_id = vocab.get(CONTENT_MODEL_END_SAMPLING) - for token_id in reversed(input_ids): - if token_id in (thinking_id, model_id): - return False - if token_id in (text_id, end_sampling_id): - return True - return False - - def count_reasoning_tokens(self, token_ids: Sequence[int]) -> int: - vocab = self.vocab - thinking_id = vocab.get(CONTENT_THINKING) - end_ids = { - token_id - for token_id in ( - vocab.get(END_MESSAGE), - vocab.get(CONTENT_MODEL_END_SAMPLING), - ) - if token_id is not None - } - in_reasoning = False - count = 0 - for token_id in token_ids: - if token_id == thinking_id: - in_reasoning = True - continue - if token_id in end_ids: - in_reasoning = False - continue - if in_reasoning: - count += 1 - return count - - def _single_pass_parse( - self, - text: str, - token_ids: Sequence[int], - initial_state: ParserState | None = None, - ) -> tuple[str | None, str | None, ExtractedToolCallInformation]: - reasoning, content, tool_call_info = super()._single_pass_parse( - text, token_ids, initial_state=initial_state - ) - # The engine defers content that follows tool-call events within a - # single pass; Inkling allows text blocks after tool-call blocks, so - # flush the trailing text (matching the Rust unified parser). - if self._deferred_content: - trailing = self._deferred_content - self._deferred_content = "" - content = self._strip_content_whitespace( - (content or "") + trailing, - tool_call_info.tools_called, - ) - tool_call_info = ExtractedToolCallInformation( - tools_called=tool_call_info.tools_called, - tool_calls=tool_call_info.tool_calls, - content=content, - ) - return reasoning, content, tool_call_info - - @staticmethod - def _extract_args_value(parsed: dict) -> str | None: - # Inkling wraps arguments under "args" rather than "arguments". - for key in ("args", "arguments", "parameters"): - if key in parsed: - val = parsed[key] - if isinstance(val, str): - return val - return json.dumps(val, ensure_ascii=False) - return None diff --git a/vllm/parser/kimi_k2.py b/vllm/parser/kimi_k2.py deleted file mode 100644 index 7c5f30aafe..0000000000 --- a/vllm/parser/kimi_k2.py +++ /dev/null @@ -1,268 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Kimi K2 parser for reasoning and tool calls. - -Kimi K2 tool call format:: - - <|tool_calls_section_begin|> - <|tool_call_begin|>functions.get_weather:0 - <|tool_call_argument_begin|>{"city": "Tokyo"}<|tool_call_end|> - <|tool_calls_section_end|> - -The header before ``<|tool_call_argument_begin|>`` is Kimi's native tool -call id. The function name is the final component before ``:N``. -""" - -from __future__ import annotations - -import functools -from collections.abc import Sequence -from typing import TYPE_CHECKING - -import regex as re - -from vllm.entrypoints.generate.base.protocol import DeltaFunctionCall, DeltaToolCall -from vllm.parser.engine.events import EventType -from vllm.parser.engine.parser_engine import ParserEngine -from vllm.parser.engine.parser_engine_config import ( - ParserEngineConfig, - ParserState, - Transition, -) - -if TYPE_CHECKING: - from vllm.entrypoints.openai.chat_completion.protocol import ( - ChatCompletionRequest, - ) - from vllm.entrypoints.openai.responses.protocol import ResponsesRequest - from vllm.tokenizers import TokenizerLike - from vllm.tool_parsers.abstract_tool_parser import Tool - -THINK_START = "" -THINK_END = "" -TOOL_SECTION_START = "<|tool_calls_section_begin|>" -TOOL_SECTION_END = "<|tool_calls_section_end|>" -TOOL_CALL_START = "<|tool_call_begin|>" -TOOL_CALL_END = "<|tool_call_end|>" -TOOL_ARG_START = "<|tool_call_argument_begin|>" - -_TOOL_ID_RE = re.compile(r"(?P.+:\d+)") - - -@functools.cache -def kimi_k2_config(thinking: bool = True) -> ParserEngineConfig: - reasoning_terminals = ( - { - "THINK_START": THINK_START, - "THINK_END": THINK_END, - } - if thinking - else {} - ) - reasoning_transitions = ( - { - (ParserState.REASONING, "THINK_START"): Transition( - ParserState.REASONING, - (), - ), - (ParserState.REASONING, "THINK_END"): Transition( - ParserState.CONTENT, - (EventType.REASONING_END,), - ), - (ParserState.CONTENT, "THINK_END"): Transition( - ParserState.CONTENT, - (), - ), - } - if thinking - else {} - ) - - return ParserEngineConfig( - name="kimi_k2", - initial_state=ParserState.REASONING if thinking else ParserState.CONTENT, - wait_for_reasoning=thinking, - terminals={ - **reasoning_terminals, - "TOOL_SECTION_START": TOOL_SECTION_START, - "TOOL_SECTION_END": TOOL_SECTION_END, - "TOOL_START": TOOL_CALL_START, - "TOOL_END": TOOL_CALL_END, - "ARG_START": TOOL_ARG_START, - }, - token_id_terminals={ - **reasoning_terminals, - "TOOL_SECTION_START": TOOL_SECTION_START, - "TOOL_SECTION_END": TOOL_SECTION_END, - "TOOL_START": TOOL_CALL_START, - "TOOL_END": TOOL_CALL_END, - "ARG_START": TOOL_ARG_START, - }, - transitions={ - **reasoning_transitions, - (ParserState.REASONING, "TOOL_SECTION_START"): Transition( - ParserState.TOOL_PREAMBLE, - (EventType.REASONING_END,), - ), - (ParserState.CONTENT, "TOOL_SECTION_START"): Transition( - ParserState.TOOL_PREAMBLE, - (), - ), - (ParserState.TOOL_PREAMBLE, "TOOL_START"): Transition( - ParserState.TOOL_NAME, - (EventType.TOOL_CALL_START,), - ), - (ParserState.TOOL_NAME, "ARG_START"): Transition( - ParserState.TOOL_ARGS, - (), - ), - (ParserState.TOOL_ARGS, "TOOL_END"): Transition( - ParserState.TOOL_BETWEEN, - (EventType.TOOL_CALL_END,), - ), - (ParserState.TOOL_ARGS, "TOOL_SECTION_END"): Transition( - ParserState.TOOL_PREAMBLE, - (EventType.TOOL_CALL_END,), - ), - (ParserState.TOOL_BETWEEN, "TOOL_START"): Transition( - ParserState.TOOL_NAME, - (EventType.TOOL_CALL_START,), - ), - # Keep the parser in a tool state after the section closes so - # trailing model text after native tool calls is suppressed. - (ParserState.TOOL_PREAMBLE, "TOOL_SECTION_END"): Transition( - ParserState.TOOL_PREAMBLE, - (), - ), - (ParserState.TOOL_BETWEEN, "TOOL_SECTION_END"): Transition( - ParserState.TOOL_PREAMBLE, - (), - ), - }, - stream_arg_deltas=True, - tool_args_json=True, - strip_trailing_reasoning_whitespace=True, - drop_whitespace_only_content_before_tools=True, - strip_content_whitespace_with_tools=False, - validate_tool_names=False, - ) - - -class KimiK2Parser(ParserEngine): - """Kimi K2 parser backed by the declarative parser engine.""" - - def __init__( - self, - tokenizer: TokenizerLike, - tools: list[Tool] | None = None, - **kwargs, - ) -> None: - chat_kwargs = kwargs.get("chat_template_kwargs", {}) or {} - thinking = chat_kwargs.get("thinking", None) - enable_thinking = chat_kwargs.get("enable_thinking", None) - self.thinking_enabled = ( - True - if thinking is None and enable_thinking is None - else bool(thinking) or bool(enable_thinking) - ) - kwargs.setdefault( - "parser_engine_config", - kimi_k2_config(thinking=self.thinking_enabled), - ) - super().__init__(tokenizer, tools, **kwargs) - - vocab = self.vocab - self._start_token_id = vocab.get(THINK_START) - self._end_token_id = vocab.get(THINK_END) - self._tool_section_start_token_id = vocab.get(TOOL_SECTION_START) - - @staticmethod - def _extract_tool_id_and_name(header: str | None) -> tuple[str | None, str | None]: - if header is None: - return None, None - match = _TOOL_ID_RE.match(header.strip()) - if not match: - return None, None - - tool_id = match.group("id").strip() - tool_name = tool_id.split(":")[0].removeprefix("functions.") - return tool_id, tool_name - - def _emit_name_delta( - self, - idx: int, - deltas: list[DeltaToolCall], - name: str | None, - ) -> None: - tool_id, tool_name = self._extract_tool_id_and_name(name) - if not tool_name: - if 0 <= idx < len(self._tool_slots): - self._tool_slots[idx].name = "" - return - - slot = self._tool_slots[idx] - slot.id = tool_id or "" - super()._emit_name_delta(idx, deltas, tool_name) - - def _handle_tool_end(self, event, deltas) -> None: - idx = event.tool_index - if 0 <= idx < len(self._tool_slots) and not self._tool_slots[idx].name_sent: - tool_id, tool_name = self._extract_tool_id_and_name( - self._tool_slots[idx].name - ) - if tool_name: - self._tool_slots[idx].id = tool_id or "" - self._tool_slots[idx].name = tool_name - super()._handle_tool_end(event, deltas) - - def _handle_arg_chunk(self, event, deltas) -> None: - idx = event.tool_index - name_sent_before = ( - 0 <= idx < len(self._tool_slots) and self._tool_slots[idx].name_sent - ) - super()._handle_arg_chunk(event, deltas) - if ( - event.value - and not name_sent_before - and 0 <= idx < len(self._tool_slots) - and self._tool_slots[idx].name_sent - ): - deltas.append( - DeltaToolCall( - index=idx, - function=DeltaFunctionCall(arguments=event.value), - ) - ) - - def _extract_args_json(self, raw_args: str, func_name: str) -> str: - return raw_args.strip() or "{}" - - def extract_content_ids(self, input_ids: list[int]) -> list[int]: - if not self.thinking_enabled: - return input_ids - - end_id = self._end_token_id - if end_id is not None and end_id in input_ids: - end_idx = len(input_ids) - 1 - input_ids[::-1].index(end_id) - return input_ids[end_idx + 1 :] - - tool_section_id = self._tool_section_start_token_id - if tool_section_id is not None and tool_section_id in input_ids: - section_idx = len(input_ids) - 1 - input_ids[::-1].index(tool_section_id) - return input_ids[section_idx:] - - return [] - - def extract_reasoning( - self, - model_output: str, - request: ChatCompletionRequest | ResponsesRequest, - ) -> tuple[str | None, str | None]: - if not self.thinking_enabled: - return None, model_output - return super().extract_reasoning(model_output, request) - - def count_reasoning_tokens(self, token_ids: Sequence[int]) -> int: - if not self.thinking_enabled: - return 0 - return super().count_reasoning_tokens(token_ids) diff --git a/vllm/parser/kimi_k3.py b/vllm/parser/kimi_k3.py deleted file mode 100644 index 587e1d5ad6..0000000000 --- a/vllm/parser/kimi_k3.py +++ /dev/null @@ -1,135 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from __future__ import annotations - -from collections.abc import Sequence -from typing import TYPE_CHECKING - -from vllm.entrypoints.generate.base.protocol import ( - DeltaMessage, - FunctionCall, -) -from vllm.parser.abstract_parser import DelegatingParser -from vllm.reasoning.kimi_k3_reasoning_parser import KimiK3ReasoningParser - -if TYPE_CHECKING: - from vllm.entrypoints.openai.chat_completion.protocol import ( - ChatCompletionRequest, - ) - from vllm.entrypoints.openai.responses.protocol import ResponsesRequest - - -class KimiK3Parser(DelegatingParser): - """Compose the Kimi K3 reasoning and tool parsers for XTML output.""" - - # TODO: Switch Kimi K3 to the parser engine once its XTML reasoning/tool - # path is covered there. - def _extract_tool_calls( - self, - content: str | None, - request: ChatCompletionRequest | ResponsesRequest, - enable_auto_tools: bool = False, - ) -> tuple[list[FunctionCall] | None, str | None]: - if self._tool_parser is None or not enable_auto_tools: - return super()._extract_tool_calls(content, request, enable_auto_tools) - - tool_call_info = self.extract_tool_calls(content or "", request=request) - if request.tool_choice == "none": - return [], tool_call_info.content - if not tool_call_info.tools_called: - return None, tool_call_info.content - - tool_calls = [ - FunctionCall( - id=tool_call.id, - name=tool_call.function.name, - arguments=tool_call.function.arguments, - ) - for tool_call in tool_call_info.tool_calls - ] - parsed_content = tool_call_info.content - if parsed_content and parsed_content.strip() == "": - parsed_content = None - return tool_calls, parsed_content - - def _extract_tool_calls_streaming( - self, - previous_text: str, - current_text: str, - delta_text: str, - previous_token_ids: Sequence[int], - current_token_ids: Sequence[int], - delta_token_ids: Sequence[int], - request: ChatCompletionRequest | ResponsesRequest, - tool_call_idx: int | None = None, - tool_call_id_type: str = "random", - function_name_returned: bool = False, - ) -> tuple[DeltaMessage | None, bool]: - if request.tool_choice != "none": - return super()._extract_tool_calls_streaming( - previous_text, - current_text, - delta_text, - previous_token_ids, - current_token_ids, - delta_token_ids, - request, - tool_call_idx=tool_call_idx, - tool_call_id_type=tool_call_id_type, - function_name_returned=function_name_returned, - ) - - delta_message = self.extract_tool_calls_streaming( - previous_text, - current_text, - delta_text, - previous_token_ids, - current_token_ids, - delta_token_ids, - request, - ) - if delta_message is not None: - delta_message.tool_calls = [] - return delta_message, False - - def parse_delta( - self, - delta_text: str, - delta_token_ids: list[int], - request: ChatCompletionRequest | ResponsesRequest, - prompt_token_ids: list[int] | None = None, - *, - finished: bool, - ) -> DeltaMessage | None: - state = self._stream_state - previous_content = state.previous_text if state.reasoning_ended else "" - delta_message = super().parse_delta( - delta_text, - delta_token_ids, - request, - prompt_token_ids, - finished=finished, - ) - - if ( - self._tool_parser is not None - or not isinstance(self._reasoning_parser, KimiK3ReasoningParser) - or not state.reasoning_ended - or delta_message is None - ): - return delta_message - - stripped = self._reasoning_parser.strip_content_streaming( - previous_text=previous_content, - current_text=state.previous_text, - ) - delta_message.content = stripped.content if stripped is not None else None - if ( - delta_message.role is None - and delta_message.content is None - and delta_message.reasoning is None - and not delta_message.tool_calls - ): - return None - return delta_message diff --git a/vllm/parser/ling3.py b/vllm/parser/ling3.py deleted file mode 100644 index 187407b409..0000000000 --- a/vllm/parser/ling3.py +++ /dev/null @@ -1,80 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Ling3 parser for reasoning and tool calls. - -Ling3 uses the same XML tool-call format as GLM-4.7 and defaults thinking on. -It follows the GLM-style ```` / ```` reasoning format and -treats ```` as an implicit reasoning terminator. -""" - -from __future__ import annotations - -from dataclasses import replace -from typing import TYPE_CHECKING - -from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest -from vllm.entrypoints.openai.responses.protocol import ResponsesRequest -from vllm.parser.engine.adapters import make_adapters -from vllm.parser.engine.parser_engine import ParserEngine -from vllm.parser.glm47_moe import ( - THINK_END, - THINK_START, - Glm47MoeParser, - glm47_moe_config, -) - -if TYPE_CHECKING: - from vllm.tokenizers import TokenizerLike - from vllm.tool_parsers.abstract_tool_parser import Tool - - -class Ling3Parser(Glm47MoeParser): - """Ling3 parser backed by the GLM XML parser engine.""" - - def __init__( - self, - tokenizer: TokenizerLike, - tools: list[Tool] | None = None, - **kwargs, - ) -> None: - chat_kwargs = kwargs.get("chat_template_kwargs", {}) or {} - thinking = chat_kwargs.get("thinking", None) - enable_thinking = chat_kwargs.get("enable_thinking", None) - self.thinking_enabled = ( - True - if thinking is None and enable_thinking is None - else bool(thinking) or bool(enable_thinking) - ) - parser_config = replace( - glm47_moe_config(thinking=self.thinking_enabled), - name="ling3", - ) - kwargs.setdefault( - "parser_engine_config", - parser_config, - ) - ParserEngine.__init__(self, tokenizer, tools, **kwargs) - - @property - def reasoning_start_str(self) -> str: - return THINK_START - - @property - def reasoning_end_str(self) -> str: - return THINK_END - - def extract_reasoning( - self, - model_output: str, - request: ChatCompletionRequest | ResponsesRequest, - ) -> tuple[str | None, str | None]: - if not self.thinking_enabled: - return None, model_output - - return super().extract_reasoning(model_output, request) - - -( - Ling3ParserReasoningAdapter, - Ling3ParserToolAdapter, -) = make_adapters(Ling3Parser) diff --git a/vllm/parser/mimo.py b/vllm/parser/mimo.py deleted file mode 100644 index 3704907efe..0000000000 --- a/vllm/parser/mimo.py +++ /dev/null @@ -1,47 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""MiMo's compact XML parameters contain verbatim string values.""" - -import functools -import json -from dataclasses import replace - -from vllm.parser.qwen3 import ( - _PARAM_RE, - _PARTIAL_PARAM_RE, - CHATML_TURN_BOUNDARIES, - Qwen3Parser, - _param_value_extender, - qwen3_config, -) - - -def _mimo_arg_converter(raw_args: str, partial: bool) -> str: - params = {m.group(1): m.group(2) for m in _PARAM_RE.finditer(raw_args)} - if partial: - match = _PARTIAL_PARAM_RE.search(_PARAM_RE.sub("", raw_args)) - if match and match.group(1): - params[match.group(1)] = match.group(2) - return json.dumps(params, ensure_ascii=False) - - -class MiMoParser(Qwen3Parser): - def __init__(self, tokenizer, tools=None, **kwargs): - thinking = (kwargs.get("chat_template_kwargs") or {}).get( - "enable_thinking", True - ) - kwargs.setdefault( - "parser_engine_config", - replace( - qwen3_config( - thinking=thinking, - name="mimo", - turn_boundary_tokens=CHATML_TURN_BOUNDARIES, - ), - arg_converter=_mimo_arg_converter, - arg_value_extender=functools.partial( - _param_value_extender, trim_newlines=False - ), - ), - ) - super().__init__(tokenizer, tools, **kwargs) diff --git a/vllm/parser/minicpmv.py b/vllm/parser/minicpmv.py deleted file mode 100644 index 88a6865c40..0000000000 --- a/vllm/parser/minicpmv.py +++ /dev/null @@ -1,405 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""MiniCPM-V family parser. - -Wraps the Qwen3 grammar (```` reasoning + XML tool calls) with two -MiniCPM-V specifics: - -- ``<|im_end|>`` and the reserved marker tokens are passthrough terminals, - so they never leak into ``content``. -- The tokenizer decodes a newline as the two characters ``\\`` and ``n`` - (e.g. token id 1639) instead of ``\\n``, so the decoded text is recovered - before it reaches the client. Code blocks, inline code and LaTeX are left - alone, as are tool call arguments. -""" - -from __future__ import annotations - -import functools -from collections.abc import Sequence -from dataclasses import replace -from typing import TYPE_CHECKING - -from vllm.entrypoints.generate.base.protocol import DeltaMessage -from vllm.parser.abstract_parser import DelegatingParser -from vllm.parser.engine.parser_engine import ParserEngine -from vllm.parser.engine.parser_engine_config import ( - ParserEngineConfig, - ParserState, - Transition, -) -from vllm.parser.qwen3 import Qwen3Parser, qwen3_config - -if TYPE_CHECKING: - from vllm.entrypoints.generate.base.protocol import FunctionCall - from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest - from vllm.entrypoints.openai.responses.protocol import ResponsesRequest - from vllm.tokenizers import TokenizerLike - from vllm.tool_parsers.abstract_tool_parser import Tool - - -_RESERVED_MARKER_IDS = range(12, 16) -_IM_END = "<|im_end|>" - - -@functools.cache -def minicpmv_config(thinking: bool) -> ParserEngineConfig: - """MiniCPM-V grammar: the qwen3 grammar plus MiniCPM-V markers. - - Both modes keep the qwen3 tool call terminals and transitions, so - ```` output is parsed whether or not thinking is enabled. - """ - name = "minicpmv" if thinking else "minicpmv_no_thinking" - base = qwen3_config(thinking=thinking, name=name) - - transitions = dict(base.transitions) - if not thinking: - # Default mode: the model answers directly, so `` must not open - # a reasoning span that would swallow the answer. A stray - # `...` in the output is still dropped, not printed. - transitions[(ParserState.CONTENT, "THINK_START")] = Transition( - ParserState.REASONING - ) - base = replace(base, strip_trailing_reasoning_whitespace=False) - - terminals = dict(base.terminals) - token_id_terminals = dict(base.token_id_terminals) - terminals["IM_END"] = _IM_END - token_id_terminals["IM_END"] = _IM_END - for state in (ParserState.CONTENT, ParserState.REASONING): - transitions[(state, "IM_END")] = Transition(state) - - for index in _RESERVED_MARKER_IDS: - text_name = f"RESERVED_TEXT_{index}" - token_name = f"RESERVED_TOKEN_{index}" - terminals[text_name] = f"" - terminals[token_name] = f"<|reserved_{index}|>" - token_id_terminals[token_name] = f"<|reserved_{index}|>" - for state in (ParserState.CONTENT, ParserState.REASONING): - transitions[(state, text_name)] = Transition(state) - transitions[(state, token_name)] = Transition(state) - - return replace( - base, - terminals=terminals, - token_id_terminals=token_id_terminals, - transitions=transitions, - ) - - -class EscapedNewlineNormalizer: - """Recover literal escape sequences written by the model as real text. - - Scans character by character so that a backslash inside a code block, - inline code, LaTeX span or a doubled escape is left untouched. A trailing - half of an escape is held in ``_pending`` until the next chunk arrives, - which keeps streaming output identical to the non-streaming result. - """ - - def __init__(self) -> None: - self._closing_marker: str | None = None - self._pending = "" - - def feed(self, text: str, *, final: bool = False) -> str: - text = self._pending + text - self._pending = "" - output: list[str] = [] - index = 0 - - while index < len(text): - if self._closing_marker is not None: - marker = self._closing_marker - remaining = text[index:] - if remaining.startswith(marker): - output.append(marker) - index += len(marker) - self._closing_marker = None - continue - if not final and marker.startswith(remaining): - self._pending = remaining - break - output.append(text[index]) - index += 1 - continue - - remaining = text[index:] - if text[index] == "`": - if not final and remaining in ("`", "``"): - self._pending = remaining - break - if remaining.startswith("```"): - output.append("```") - index += 3 - self._closing_marker = "```" - else: - output.append("`") - index += 1 - self._closing_marker = "`" - continue - - if text[index] == "$": - if not final and remaining == "$": - self._pending = remaining - break - if remaining.startswith("$$"): - output.append("$$") - index += 2 - self._closing_marker = "$$" - else: - output.append("$") - index += 1 - self._closing_marker = "$" - continue - - if text[index] != "\\": - output.append(text[index]) - index += 1 - continue - - slash_end = index + 1 - while slash_end < len(text) and text[slash_end] == "\\": - slash_end += 1 - if slash_end - index > 1: - if not final and slash_end == len(text): - # The run may still grow, and its parity decides whether a - # following `n` is a newline or stays literal. Hold it until - # the run is complete. - self._pending = text[index:slash_end] - break - output.append(text[index:slash_end]) - index = slash_end - continue - - if not final and remaining in ("\\", "\\r", "\\r\\"): - self._pending = remaining - break - if remaining.startswith("\\r\\n"): - output.append("\n") - index += 4 - elif remaining.startswith(("\\n", "\\r")): - output.append("\n") - index += 2 - elif remaining.startswith("\\("): - output.append("\\(") - index += 2 - self._closing_marker = "\\)" - elif remaining.startswith("\\["): - output.append("\\[") - index += 2 - self._closing_marker = "\\]" - else: - output.append("\\") - index += 1 - - if final and self._pending: - output.append(self._pending) - self._pending = "" - return "".join(output) - - -def recover_newlines(text: str | None) -> str | None: - if text is None or "\\" not in text: - return text - return EscapedNewlineNormalizer().feed(text, final=True) - - -class MiniCPMVOutputNormalizer: - """Apply :class:`EscapedNewlineNormalizer` to a whole stream. - - Reasoning and content are tracked separately so that a half-written escape - at the end of a reasoning chunk does not swallow the first characters of - the content that follows. - """ - - def __init__(self) -> None: - self._reasoning = EscapedNewlineNormalizer() - self._content = EscapedNewlineNormalizer() - - def normalize_delta( - self, - delta: DeltaMessage | None, - *, - finished: bool, - ) -> DeltaMessage | None: - reasoning_finished = finished or ( - delta is not None and (delta.content is not None or bool(delta.tool_calls)) - ) - reasoning = self._reasoning.feed( - delta.reasoning if delta and delta.reasoning else "", - final=reasoning_finished, - ) - content = self._content.feed( - delta.content if delta and delta.content else "", - final=finished, - ) - - if delta is None: - if not reasoning and not content: - return None - return DeltaMessage( - reasoning=reasoning or None, - content=content or None, - ) - - if reasoning: - delta.reasoning = reasoning - elif delta.reasoning is not None: - delta.reasoning = None - if content: - delta.content = content - elif delta.content is not None: - delta.content = None - - # A chunk that normalizes to nothing must be dropped, not returned as an - # all-None DeltaMessage: the engine only skips on `None`, so anything - # else reaches the client as an empty delta. - if not delta.content and not delta.reasoning and not delta.tool_calls: - return None - return delta - - -class MiniCPMVParser(Qwen3Parser): - """Parse MiniCPM-V family output in thinking and non-thinking modes.""" - - CONFIG_NAME = "minicpmv" - - def __init__( - self, - tokenizer: TokenizerLike, - tools: list[Tool] | None = None, - **kwargs, - ) -> None: - chat_kwargs = kwargs.get("chat_template_kwargs", {}) or {} - # The chat template emits `` unless `enable_thinking` is - # explicitly false, so the grammar must default to thinking too, or the - # reasoning span is misread as content. - thinking_enabled = chat_kwargs.get("enable_thinking", True) - kwargs.setdefault("parser_engine_config", minicpmv_config(thinking_enabled)) - super().__init__(tokenizer, tools, **kwargs) - # ``Qwen3Parser.__init__`` reads the same key with a default of True, - # which does not apply here: MiniCPM-V defaults to thinking disabled. - # Keep the attribute in sync with the grammar built above. - self.thinking_enabled = thinking_enabled - - @property - def reasoning_ended(self) -> bool: - # The engine derives this from the presence of THINK terminals, which - # both MiniCPM-V configs keep, so the base property stays False in - # non-thinking mode. There is no reasoning span to end there. - if not self.thinking_enabled: - return True - return super().reasoning_ended - - def extract_reasoning( - self, - model_output: str, - request: ChatCompletionRequest | ResponsesRequest, - ) -> tuple[str | None, str | None]: - if self.thinking_enabled: - return super().extract_reasoning(model_output, request) - _, content = ParserEngine.extract_reasoning(self, model_output, request) - return None, content - - def extract_reasoning_streaming( - self, - previous_text: str, - current_text: str, - delta_text: str, - previous_token_ids: Sequence[int], - current_token_ids: Sequence[int], - delta_token_ids: Sequence[int], - ) -> DeltaMessage | None: - if self.thinking_enabled: - return super().extract_reasoning_streaming( - previous_text, - current_text, - delta_text, - previous_token_ids, - current_token_ids, - delta_token_ids, - ) - - delta = ParserEngine.extract_reasoning_streaming( - self, - previous_text, - current_text, - delta_text, - previous_token_ids, - current_token_ids, - delta_token_ids, - ) - if delta is None: - return None - delta.reasoning = None - if delta.content is None and not delta.tool_calls: - return None - return delta - - def get_streaming_fallback_content( - self, - text: str, - request: ChatCompletionRequest | ResponsesRequest, - ) -> str | None: - if self.thinking_enabled: - return super().get_streaming_fallback_content(text, request) - delta = ParserEngine.finish_streaming(self) - return delta.content if delta is not None else None - - -class MiniCPMVUnifiedParser(DelegatingParser): - """MiniCPM-V serving parser: the adapters plus newline recovery. - - Recovery cannot live in :class:`MiniCPMVParser`: the serving stack drives - the reasoning/tool adapters, never ``ParserEngine.parse`` / - ``ParserEngine.parse_delta``, so a rewrite applied there would only be - visible to callers that construct the engine directly. - """ - - def __init__( - self, - tokenizer: TokenizerLike, - tools: list[Tool] | None = None, - *args, - **kwargs, - ) -> None: - super().__init__(tokenizer, tools, *args, **kwargs) - self._output_normalizer = MiniCPMVOutputNormalizer() - - def parse( - self, - model_output: str, - request: ChatCompletionRequest | ResponsesRequest, - enable_auto_tools: bool = False, - model_output_token_ids: Sequence[int] = (), - ) -> tuple[str | None, str | None, list[FunctionCall] | None]: - reasoning, content, tool_calls = super().parse( - model_output, - request, - enable_auto_tools, - model_output_token_ids, - ) - # Tool call arguments are JSON, where a literal `\n` is meaningful and - # must survive as written. - return ( - recover_newlines(reasoning), - recover_newlines(content), - tool_calls, - ) - - def parse_delta( - self, - delta_text: str, - delta_token_ids: list[int], - request: ChatCompletionRequest | ResponsesRequest, - prompt_token_ids: list[int] | None = None, - *, - finished: bool, - ) -> DeltaMessage | None: - delta = super().parse_delta( - delta_text, - delta_token_ids, - request, - prompt_token_ids, - finished=finished, - ) - return self._output_normalizer.normalize_delta(delta, finished=finished) diff --git a/vllm/parser/minimax_m2.py b/vllm/parser/minimax_m2.py deleted file mode 100644 index d804148ec7..0000000000 --- a/vllm/parser/minimax_m2.py +++ /dev/null @@ -1,265 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""MiniMax M2 parser for XML-style tool calls. - -MiniMax M2 tool call format:: - - - Seattle - - -Each ```` block becomes one tool call. The argument body consists -of ``...`` tags. -""" - -from __future__ import annotations - -import functools -import json - -import regex as re - -from vllm.parser.engine.events import EventType -from vllm.parser.engine.parser_engine import ParserEngine -from vllm.parser.engine.parser_engine_config import ( - ArgValueExtender, - ParserEngineConfig, - ParserState, - Transition, -) - -TOOL_CALL_START = "" -TOOL_CALL_END = "" -THINK_START = "" -THINK_END = "" -INVOKE_PREFIX_DQ = '' -NAME_END_SQ = "'>" -NAME_END_UNQUOTED = ">" -PARAM_START = "" -_PARAM_RE = re.compile( - r"<\s*parameter\s+name\s*=\s*" - r"(?:\"(?P[^\"]*)\"|'(?P[^']*)'|(?P[^>\s]+))" - r"\s*>" - r"(?P.*?)" - r"(?:<\s*/\s*parameter\s*>|(?=<\s*parameter\s+name\s*=))", - re.DOTALL, -) -_PARTIAL_PARAM_RE = re.compile( - r"<\s*parameter\s+name\s*=\s*" - r"(?:\"(?P[^\"]*)\"|'(?P[^']*)'|(?P[^>\s]+))" - r"\s*>" - r"(?P.*)$", - re.DOTALL, -) - - -# What ends an open value in ``_PARAM_RE``: a closing tag or the next opener. -_PARAM_VALUE_END_RE = re.compile(r"<\s*/\s*parameter\s*>|<\s*parameter\s+name\s*=") - - -def _param_name(match: re.Match) -> str: - return ( - match.group("dq_name") - or match.group("sq_name") - or match.group("bare_name") - or "" - ).strip() - - -def _minimax_m2_parse_args( - raw_args: str, partial: bool -) -> tuple[dict[str, object], re.Match | None]: - """Return the arguments and the match of the open trailing value, if any.""" - params: dict[str, object] = {} - - for match in _PARAM_RE.finditer(raw_args): - name = _param_name(match) - if not name: - continue - # Keep the value verbatim (like the glm47 converter): it is inline - # between `>` and ``, so surrounding whitespace is data. - params[name] = match.group("value") - - open_match = None - if partial: - remaining = _PARAM_RE.sub("", raw_args) - match = _PARTIAL_PARAM_RE.search(remaining) - if match: - name = _param_name(match) - if name: - # Verbatim, same as the complete-match loop above. - params[name] = match.group("value") - open_match = match - - return params, open_match - - -def _minimax_m2_arg_converter(raw_args: str, partial: bool) -> str: - return json.dumps(_minimax_m2_parse_args(raw_args, partial)[0], ensure_ascii=False) - - -def _value_end_window(text: str) -> str: - """Return the suffix of *text* that may still grow into a value end.""" - start = text.rfind("<") - if start == -1: - return "" - tail = text[start:] - return tail if _PARAM_VALUE_END_RE.fullmatch(tail, partial=True) else "" - - -def _minimax_m2_arg_value_extender(raw_args: str) -> ArgValueExtender | None: - # The open value runs to the end of raw_args and is the last JSON value, - # so a delta appends its JSON-escaped text unless it closes the value. - params, match = _minimax_m2_parse_args(raw_args, partial=True) - if match is None or next(reversed(params)) != _param_name(match): - return None - # Later text could still re-parse the opener: an earlier unmatched one, or - # a bare name that starts a quote, may close into a different name. - bare_name = match.group("bare_name") - if _PARAM_VALUE_END_RE.search(match.string, 0, match.start()) or ( - bare_name is not None and bare_name[0] in "\"'" - ): - return None - - window = _value_end_window(match.group("value")) - - def extend(delta: str) -> str | None: - nonlocal window - text = window + delta - if _PARAM_VALUE_END_RE.search(text): - return None - window = _value_end_window(text) - return json.dumps(delta, ensure_ascii=False)[1:-1] - - return extend - - -@functools.cache -def minimax_m2_config() -> ParserEngineConfig: - return ParserEngineConfig( - name="minimax_m2", - initial_state=ParserState.REASONING, - wait_for_reasoning=True, - terminals={ - "THINK_START": THINK_START, - "THINK_END": THINK_END, - "TOOL_START": TOOL_CALL_START, - "PARAM_START": PARAM_START, - "PARAM_END": PARAM_END, - "TOOL_END": TOOL_CALL_END, - "INVOKE_PREFIX_DQ": INVOKE_PREFIX_DQ, - "INVOKE_PREFIX_SQ": INVOKE_PREFIX_SQ, - "INVOKE_PREFIX_UNQUOTED": INVOKE_PREFIX_UNQUOTED, - "INVOKE_END": INVOKE_END, - "NAME_END_DQ": NAME_END_DQ, - "NAME_END_SQ": NAME_END_SQ, - "NAME_END_UNQUOTED": NAME_END_UNQUOTED, - }, - token_id_terminals={ - "THINK_START": THINK_START, - "THINK_END": THINK_END, - "TOOL_START": TOOL_CALL_START, - "TOOL_END": TOOL_CALL_END, - }, - transitions={ - (ParserState.REASONING, "THINK_START"): Transition( - ParserState.REASONING, - (), - ), - (ParserState.REASONING, "THINK_END"): Transition( - ParserState.CONTENT, - (EventType.REASONING_END,), - ), - (ParserState.CONTENT, "THINK_END"): Transition( - ParserState.CONTENT, - (), - ), - (ParserState.REASONING, "TOOL_START"): Transition( - ParserState.TOOL_PREAMBLE, - (EventType.REASONING_END,), - ), - (ParserState.CONTENT, "TOOL_START"): Transition( - ParserState.TOOL_PREAMBLE, - (), - ), - (ParserState.TOOL_ARGS, "PARAM_START"): Transition( - ParserState.TOOL_ARGS, - (EventType.ARG_VALUE_CHUNK,), - ), - (ParserState.TOOL_ARGS, "PARAM_END"): Transition( - ParserState.TOOL_ARGS, - (EventType.ARG_VALUE_CHUNK,), - ), - (ParserState.TOOL_PREAMBLE, "TOOL_END"): Transition( - ParserState.CONTENT, - (), - ), - (ParserState.TOOL_BETWEEN, "TOOL_END"): Transition( - ParserState.CONTENT, - (), - ), - (ParserState.CONTENT, "TOOL_END"): Transition( - ParserState.CONTENT, - (), - ), - (ParserState.TOOL_ARGS, "INVOKE_END"): Transition( - ParserState.TOOL_BETWEEN, - (EventType.TOOL_CALL_END,), - ), - **{ - (state, terminal): Transition( - ParserState.TOOL_NAME, - (EventType.TOOL_CALL_START,), - ) - for state in ( - ParserState.CONTENT, - ParserState.TOOL_PREAMBLE, - ParserState.TOOL_BETWEEN, - ) - for terminal in ( - "INVOKE_PREFIX_DQ", - "INVOKE_PREFIX_SQ", - "INVOKE_PREFIX_UNQUOTED", - ) - }, - **{ - (ParserState.TOOL_NAME, terminal): Transition( - ParserState.TOOL_ARGS, - (), - ) - for terminal in ( - "NAME_END_DQ", - "NAME_END_SQ", - "NAME_END_UNQUOTED", - ) - }, - }, - arg_converter=_minimax_m2_arg_converter, - arg_value_extender=_minimax_m2_arg_value_extender, - stream_arg_deltas=True, - tool_args_json=False, - validate_tool_names=True, - ) - - -class MinimaxM2Parser(ParserEngine): - """MiniMax M2 parser backed by the declarative parser engine.""" - - def __init__(self, tokenizer, tools=None, **kwargs) -> None: - kwargs.setdefault("parser_engine_config", minimax_m2_config()) - super().__init__(tokenizer, tools, **kwargs) - self._think_end_token_id = self.vocab.get(THINK_END) - - def extract_content_ids(self, input_ids: list[int]) -> list[int]: - end_id = self._think_end_token_id - if end_id is None: - return [] - for i in range(len(input_ids) - 1, -1, -1): - if input_ids[i] == end_id: - return input_ids[i + 1 :] - return [] diff --git a/vllm/parser/minimax_m3.py b/vllm/parser/minimax_m3.py deleted file mode 100644 index dafe1457e9..0000000000 --- a/vllm/parser/minimax_m3.py +++ /dev/null @@ -1,293 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""MiniMax M3 parser for namespace-delimited XML-style tool calls. - -MiniMax M3 prefixes every structural tag with the ``]<]minimax[>[`` marker:: - - ]<]minimax[>[ - ]<]minimax[>[ - ]<]minimax[>[42]<]minimax[>[ - ]<]minimax[>[ - ]<]minimax[>[]<]minimax[>[book-001]<]minimax[>[]<]minimax[>[ - ]<]minimax[>[ - ]<]minimax[>[ - ]<]minimax[>[ - -Each ```` becomes one tool call. Parameters are elements named after -the argument and may nest: an element holding child elements becomes an -object, or an array when its schema says so (the chat template renders list -items as ````). Arguments are emitted once the whole ```` is -parsed. Reasoning is handled by ``MiniMaxM3ReasoningParser``. -""" - -from __future__ import annotations - -import functools -import json -from typing import TYPE_CHECKING, Any - -from vllm.parser.engine.events import EventType -from vllm.parser.engine.parser_engine import ParserEngine -from vllm.parser.engine.parser_engine_config import ( - ParserEngineConfig, - ParserState, - Transition, -) -from vllm.tool_parsers.utils import ( - coerce_to_schema_type, - extract_types_from_schema, - find_tool_properties, -) - -if TYPE_CHECKING: - from vllm.entrypoints.openai.chat_completion.protocol import ( - ChatCompletionRequest, - ) - from vllm.entrypoints.openai.responses.protocol import ResponsesRequest - from vllm.tokenizers import TokenizerLike - from vllm.tool_parsers.abstract_tool_parser import Tool - -NAMESPACE = "]<]minimax[>[" -TOOL_CALL_START = f"{NAMESPACE}" -TOOL_CALL_END = f"{NAMESPACE}" -INVOKE_PREFIX_DQ = f'{NAMESPACE}' -NAME_END_SQ = "'>" -NAME_END_UNQUOTED = ">" -ELEMENT_START = f"{NAMESPACE}<" -ELEMENT_END_START = f"{NAMESPACE} tuple[str, _Element, int] | None: - """Parse the element starting at ``pos``; ``None`` if malformed.""" - if depth > MAX_ELEMENT_DEPTH: - return None - name_start = pos + len(ELEMENT_START) - name_end = text.find(">", name_start) - if name_end <= name_start: - return None - name = text[name_start:name_end] - if name.startswith("/") or not name.strip(): - return None - - close_tag = f"{ELEMENT_END_START}{name}>" - pos = name_end + 1 - text_parts: list[str] = [] - children: list[tuple[str, _Element]] = [] - while True: - marker = text.find(NAMESPACE, pos) - if marker < 0: - return None - text_parts.append(text[pos:marker]) - if text.startswith(close_tag, marker): - pos = marker + len(close_tag) - break - if not text.startswith(ELEMENT_START, marker): - return None - child = _parse_element(text, marker, depth + 1) - if child is None: - return None - child_name, child_value, pos = child - children.append((child_name, child_value)) - - body_text = "".join(text_parts) - if not children: - return name, body_text, pos - if body_text.strip(): - # Keep mixed text under a reserved field that no child uses. - field = MIXED_TEXT_FIELD - while any(child_name == field for child_name, _ in children): - field = "$" + field - children.append((field, body_text)) - return name, children, pos - - -def _parse_invoke_params(raw_args: str) -> list[tuple[str, _Element]]: - """Parse the parameter elements of one ```` body. - - Parsing stops at the first malformed or unterminated element, keeping the - parameters before it. - """ - params: list[tuple[str, _Element]] = [] - pos = 0 - while True: - while pos < len(raw_args) and raw_args[pos].isspace(): - pos += 1 - if not raw_args.startswith(ELEMENT_START, pos): - return params - element = _parse_element(raw_args, pos, depth=1) - if element is None: - return params - name, value, pos = element - params.append((name, value)) - - -def _container_schema(schema: dict[str, Any]) -> tuple[str, dict[str, Any]] | None: - """Return the first ``array`` or ``object`` alternative of ``schema``.""" - types = schema.get("type") - for kind in ("array", "object"): - if types == kind or (isinstance(types, list) and kind in types): - return kind, schema - if types is None: - if "items" in schema: - return "array", schema - if "properties" in schema or "additionalProperties" in schema: - return "object", schema - for choice_field in ("anyOf", "oneOf"): - for choice in schema.get(choice_field) or (): - if isinstance(choice, dict) and (found := _container_schema(choice)): - return found - return None - - -def _convert_element(value: _Element, schema: Any) -> Any: - if not isinstance(schema, dict): - schema = None - container = _container_schema(schema) if schema is not None else None - - if isinstance(value, str): - if schema is None: - return value - types = extract_types_from_schema(schema) - # The template renders an empty list or dict as an empty element. - if container is not None and "string" not in types and not value.strip(): - return [] if container[0] == "array" else {} - return coerce_to_schema_type(value, types) - - if container is not None and container[0] == "array": - items = container[1].get("items") - return [_convert_element(child, items) for _, child in value] - return _convert_object(value, container[1] if container is not None else None) - - -def _convert_object( - elements: list[tuple[str, _Element]], schema: dict[str, Any] | None -) -> dict[str, Any]: - """Convert named elements to an object; repeated names collect into a list.""" - properties = (schema or {}).get("properties") or {} - additional = (schema or {}).get("additionalProperties") - result: dict[str, Any] = {} - repeated: set[str] = set() - for name, element in elements: - converted = _convert_element(element, properties.get(name, additional)) - if name not in result: - result[name] = converted - elif name in repeated: - result[name].append(converted) - else: - result[name] = [result[name], converted] - repeated.add(name) - return result - - -def _minimax_m3_arg_converter( - raw_args: str, - partial: bool, - properties: dict[str, Any] | None = None, -) -> str: - params = _parse_invoke_params(raw_args) - arguments = _convert_object(params, {"properties": properties or {}}) - return json.dumps(arguments, ensure_ascii=False) - - -@functools.cache -def minimax_m3_config() -> ParserEngineConfig: - return ParserEngineConfig( - name="minimax_m3", - initial_state=ParserState.CONTENT, - terminals={ - "TOOL_START": TOOL_CALL_START, - "TOOL_END": TOOL_CALL_END, - "INVOKE_PREFIX_DQ": INVOKE_PREFIX_DQ, - "INVOKE_PREFIX_SQ": INVOKE_PREFIX_SQ, - "INVOKE_PREFIX_UNQUOTED": INVOKE_PREFIX_UNQUOTED, - "INVOKE_END": INVOKE_END, - "NAME_END_DQ": NAME_END_DQ, - "NAME_END_SQ": NAME_END_SQ, - "NAME_END_UNQUOTED": NAME_END_UNQUOTED, - }, - transitions={ - (ParserState.CONTENT, "TOOL_START"): Transition( - ParserState.TOOL_PREAMBLE, - (), - ), - **{ - (state, terminal): Transition( - ParserState.TOOL_NAME, - (EventType.TOOL_CALL_START,), - ) - for state in (ParserState.TOOL_PREAMBLE, ParserState.TOOL_BETWEEN) - for terminal in ( - "INVOKE_PREFIX_DQ", - "INVOKE_PREFIX_SQ", - "INVOKE_PREFIX_UNQUOTED", - ) - }, - **{ - (ParserState.TOOL_NAME, terminal): Transition( - ParserState.TOOL_ARGS, - (), - ) - for terminal in ("NAME_END_DQ", "NAME_END_SQ", "NAME_END_UNQUOTED") - }, - # Nested parameter elements are argument text; only the invoke - # closer ends the call. - (ParserState.TOOL_ARGS, "INVOKE_END"): Transition( - ParserState.TOOL_BETWEEN, - (EventType.TOOL_CALL_END,), - ), - # The tool block ends the message: stay in TOOL_BETWEEN so any - # text after it is dropped. - (ParserState.TOOL_PREAMBLE, "TOOL_END"): Transition( - ParserState.TOOL_BETWEEN, - (), - ), - (ParserState.TOOL_BETWEEN, "TOOL_END"): Transition( - ParserState.TOOL_BETWEEN, - (), - ), - }, - arg_converter=_minimax_m3_arg_converter, - # Emit arguments once per ``: converted values are not - # prefix-stable while elements arrive, e.g. a repeated parameter name - # turns an already streamed scalar into a list. - stream_arg_deltas=False, - tool_args_json=False, - strip_content_whitespace_with_tools=False, - ) - - -class MinimaxM3Parser(ParserEngine): - """MiniMax M3 tool-call parser backed by the declarative parser engine.""" - - def __init__( - self, - tokenizer: TokenizerLike, - tools: list[Tool] | None = None, - **kwargs, - ) -> None: - kwargs.setdefault("parser_engine_config", minimax_m3_config()) - super().__init__(tokenizer, tools, **kwargs) - self._arg_converter = self._convert_args - - def adjust_request( - self, request: ChatCompletionRequest | ResponsesRequest - ) -> ChatCompletionRequest | ResponsesRequest: - # The M3 markers are non-special added tokens, so they decode with - # special tokens skipped. - return request - - def _convert_args(self, raw_args: str, partial: bool) -> str: - func_name = next( - (slot.name for slot in self._tool_slots if slot.args == raw_args), "" - ) - properties = find_tool_properties(self._tools, func_name.strip()) - return _minimax_m3_arg_converter(raw_args, partial, properties) diff --git a/vllm/parser/nemotron_v3.py b/vllm/parser/nemotron_v3.py deleted file mode 100644 index c8b75a2c7d..0000000000 --- a/vllm/parser/nemotron_v3.py +++ /dev/null @@ -1,113 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Nemotron V3 parser. - -The Nemotron 3 Super model uses the same tool call and reasoning -format as Qwen3 (````/```` + ```` XML). -This config reuses :func:`qwen3_config` with a distinct name. - -When ``enable_thinking=False`` or ``force_nonempty_content=True`` and -content is empty, reasoning and content are swapped. -""" - -from __future__ import annotations - -import dataclasses -import functools -from typing import TYPE_CHECKING - -from vllm.parser.qwen3 import CHATML_TURN_BOUNDARIES, Qwen3Parser, qwen3_config - -if TYPE_CHECKING: - from vllm.entrypoints.generate.base.protocol import DeltaMessage - from vllm.entrypoints.openai.chat_completion.protocol import ( - ChatCompletionRequest, - ) - from vllm.entrypoints.openai.responses.protocol import ResponsesRequest - from vllm.parser.engine.parser_engine import SemanticEvent - from vllm.parser.engine.parser_engine_config import ParserEngineConfig - from vllm.tokenizers import TokenizerLike - from vllm.tool_parsers.abstract_tool_parser import Tool - - -@functools.cache -def nemotron_v3_config(thinking: bool = True) -> ParserEngineConfig: - return dataclasses.replace( - qwen3_config(thinking=thinking, turn_boundary_tokens=CHATML_TURN_BOUNDARIES), - name="nemotron_v3", - strip_trailing_reasoning_whitespace=True, - ) - - -class NemotronV3Parser(Qwen3Parser): - """Nemotron V3 parser: same format as Qwen3, with Nemotron-specific - behavior: when ``enable_thinking=False`` or - ``force_nonempty_content=True`` and content is empty, swaps - reasoning and content. - """ - - def __init__( - self, - tokenizer: TokenizerLike, - tools: list[Tool] | None = None, - **kwargs, - ) -> None: - chat_kwargs = kwargs.get("chat_template_kwargs", {}) or {} - thinking = chat_kwargs.get("enable_thinking", True) - super().__init__( - tokenizer, - tools, - parser_engine_config=nemotron_v3_config(thinking=thinking), - **kwargs, - ) - self._streamed_reasoning: list[str] = [] - - def _reset(self, initial_state=None) -> None: - super()._reset(initial_state=initial_state) - self._streamed_reasoning = [] - - def _events_to_delta( - self, - events: list[SemanticEvent], - finished: bool = False, - ) -> DeltaMessage | None: - delta = super()._events_to_delta(events, finished=finished) - if delta is not None and delta.reasoning is not None: - self._streamed_reasoning.append(delta.reasoning) - return delta - - @staticmethod - def _should_force_content( - request: ChatCompletionRequest | ResponsesRequest, - ) -> bool: - chat_template_kwargs = getattr(request, "chat_template_kwargs", None) - return bool( - chat_template_kwargs - and ( - chat_template_kwargs.get("enable_thinking") is False - or chat_template_kwargs.get("force_nonempty_content") is True - ) - ) - - def get_streaming_fallback_content( - self, - text: str, - request: ChatCompletionRequest | ResponsesRequest, - ) -> str | None: - if not self._should_force_content(request): - return None - return "".join(self._streamed_reasoning) or None - - def extract_reasoning( - self, - model_output: str, - request: ChatCompletionRequest | ResponsesRequest, - ) -> tuple[str | None, str | None]: - reasoning, content = super().extract_reasoning(model_output, request) - - if self._should_force_content(request) and ( - content is None or not content.strip() - ): - reasoning, content = content, reasoning - - return reasoning, content diff --git a/vllm/parser/plamo3.py b/vllm/parser/plamo3.py deleted file mode 100644 index 8f37cc9b56..0000000000 --- a/vllm/parser/plamo3.py +++ /dev/null @@ -1,318 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from __future__ import annotations - -import functools -import json -from dataclasses import replace -from itertools import groupby -from typing import TYPE_CHECKING - -from vllm.entrypoints.generate.base.protocol import ( - DeltaFunctionCall, - DeltaMessage, - DeltaToolCall, -) -from vllm.parser.engine.events import EventType, SemanticEvent -from vllm.parser.engine.parser_engine import ParserEngine -from vllm.parser.engine.parser_engine_config import ( - ParserEngineConfig, - ParserState, - Transition, -) - -if TYPE_CHECKING: - from vllm.entrypoints.openai.chat_completion.protocol import ( - ChatCompletionRequest, - ) - from vllm.entrypoints.openai.responses.protocol import ResponsesRequest - from vllm.tokenizers import TokenizerLike - from vllm.tool_parsers.abstract_tool_parser import Tool - -BEGIN_TOOL_REQUESTS = "<|plamo:begin_tool_requests:plamo|>" -END_TOOL_REQUESTS = "<|plamo:end_tool_requests:plamo|>" -BEGIN_TOOL_REQUEST = "<|plamo:begin_tool_request:plamo|>" -END_TOOL_REQUEST = "<|plamo:end_tool_request:plamo|>" -BEGIN_TOOL_NAME = "<|plamo:begin_tool_name:plamo|>" -END_TOOL_NAME = "<|plamo:end_tool_name:plamo|>" -BEGIN_TOOL_ARGUMENTS = ( - "<|plamo:begin_tool_arguments:plamo|><|plamo:constrain|>json<|plamo:msg|>" -) -END_TOOL_ARGUMENTS = "<|plamo:end_tool_arguments:plamo|>" -BEGIN_THINK = "<|plamo:begin_think:plamo|>" -END_THINK = "<|plamo:end_think:plamo|>" -EOT = "<|plamo:tag|>" - -# PLaMo3 control markers are split into multiple tokenizer tokens. These -# atomic pieces must not be treated as unrelated special tokens by the -# parser-engine drop-token handling. -PLAMO_MARKER_TOKENS = frozenset( - { - "<|plamo:begin_", - "<|plamo:end_", - ":plamo|>", - } -) - - -@functools.cache -def plamo3_config(thinking: bool = True) -> ParserEngineConfig: - if thinking: - reasoning_terminals = { - "THINK_START": BEGIN_THINK, - "THINK_END": END_THINK, - } - reasoning_transitions = { - (ParserState.REASONING, "THINK_START"): Transition( - ParserState.REASONING, - ), - (ParserState.CONTENT, "THINK_START"): Transition( - ParserState.REASONING, - (EventType.REASONING_START,), - ), - (ParserState.REASONING, "THINK_END"): Transition( - ParserState.CONTENT, - (EventType.REASONING_END,), - ), - (ParserState.CONTENT, "THINK_END"): Transition( - ParserState.CONTENT, - (), - ), - (ParserState.REASONING, "TOOL_REQUESTS_START"): Transition( - ParserState.TOOL_PREAMBLE, - (EventType.REASONING_END,), - ), - } - initial_state = ParserState.REASONING - else: - reasoning_terminals = {} - reasoning_transitions = {} - initial_state = ParserState.CONTENT - - return ParserEngineConfig( - name="plamo3", - initial_state=initial_state, - terminals={ - **reasoning_terminals, - "TOOL_REQUESTS_START": BEGIN_TOOL_REQUESTS, - "TOOL_REQUESTS_END": END_TOOL_REQUESTS, - "TOOL_REQUEST_START": BEGIN_TOOL_REQUEST, - "TOOL_REQUEST_END": END_TOOL_REQUEST, - "TOOL_NAME_START": BEGIN_TOOL_NAME, - "TOOL_NAME_END": END_TOOL_NAME, - "TOOL_ARGS_START": BEGIN_TOOL_ARGUMENTS, - "TOOL_ARGS_END": END_TOOL_ARGUMENTS, - "EOT": EOT, - }, - transitions={ - **reasoning_transitions, - (ParserState.CONTENT, "TOOL_REQUESTS_START"): Transition( - ParserState.TOOL_PREAMBLE, - ), - (ParserState.TOOL_PREAMBLE, "TOOL_REQUEST_START"): Transition( - ParserState.TOOL_PREAMBLE, - ), - (ParserState.TOOL_PREAMBLE, "TOOL_NAME_START"): Transition( - ParserState.TOOL_NAME, - (EventType.TOOL_CALL_START,), - ), - (ParserState.TOOL_NAME, "TOOL_NAME_END"): Transition( - ParserState.TOOL_PREAMBLE, - ), - (ParserState.TOOL_PREAMBLE, "TOOL_ARGS_START"): Transition( - ParserState.TOOL_ARGS, - ), - (ParserState.TOOL_ARGS, "TOOL_ARGS_END"): Transition( - ParserState.TOOL_BETWEEN, - ), - (ParserState.TOOL_BETWEEN, "TOOL_REQUEST_END"): Transition( - ParserState.TOOL_PREAMBLE, - (EventType.TOOL_CALL_END,), - ), - (ParserState.TOOL_PREAMBLE, "TOOL_REQUESTS_END"): Transition( - ParserState.CONTENT, - ), - # EOT is normally consumed by generation stop handling. Keep it - # in the grammar as well so text-only parsing and explicitly - # supplied stop tokens cannot leak it into user-visible output. - (ParserState.REASONING, "EOT"): Transition( - ParserState.CONTENT, - (EventType.REASONING_END,), - ), - (ParserState.CONTENT, "EOT"): Transition(ParserState.CONTENT), - (ParserState.TOOL_PREAMBLE, "EOT"): Transition(ParserState.CONTENT), - (ParserState.TOOL_NAME, "EOT"): Transition(ParserState.CONTENT), - (ParserState.TOOL_ARGS, "EOT"): Transition(ParserState.CONTENT), - (ParserState.TOOL_BETWEEN, "EOT"): Transition(ParserState.CONTENT), - }, - token_id_terminals={"EOT": EOT, **reasoning_terminals}, - drop_whitespace_only_content_before_tools=True, - stream_arg_deltas=True, - strip_content_whitespace_with_tools=False, - strip_trailing_reasoning_whitespace=False, - preserve_tokens=PLAMO_MARKER_TOKENS, - tool_args_json=True, - ) - - -class Plamo3Parser(ParserEngine): - """PLaMo3 reasoning and tool-call parser backed by ParserEngine.""" - - def __init__( - self, - tokenizer: TokenizerLike, - tools: list[Tool] | None = None, - **kwargs, - ) -> None: - # Configure PLaMo parsing and cache its multi-token reasoning markers. - chat_kwargs = kwargs.get("chat_template_kwargs", {}) or {} - self.thinking_enabled = chat_kwargs.get("enable_thinking", True) - kwargs.setdefault( - "parser_engine_config", - plamo3_config(thinking=self.thinking_enabled), - ) - super().__init__(tokenizer, tools, **kwargs) - - self._reasoning_start_token_id_sequence: list[int] = list( - tokenizer.encode(BEGIN_THINK, add_special_tokens=False) - ) - self._reasoning_end_token_id_sequence: list[int] = list( - tokenizer.encode(END_THINK, add_special_tokens=False) - ) - # Token-ID sequences of every terminal literal, for token-level - # unfinished-marker detection. - self._terminal_token_id_sequences: dict[str, tuple[int, ...]] = { - literal: tuple(tokenizer.encode(literal, add_special_tokens=False)) - for literal in self.parser_engine_config.terminal_literals - } - - def is_reasoning_end(self, input_ids: list[int]) -> bool: - # Detect PLaMo reasoning boundaries that span multiple token IDs. - if not self.thinking_enabled: - return True - - start_ids = self._reasoning_start_token_id_sequence - end_ids = self._reasoning_end_token_id_sequence - for i in range(len(input_ids) - 1, -1, -1): - if input_ids[i : i + len(end_ids)] == end_ids: - return True - if input_ids[i : i + len(start_ids)] == start_ids: - return False - if input_ids[i] == self.vocab.get(EOT): - return False - return False - - def extract_content_ids(self, input_ids: list[int]) -> list[int]: - # Extract content after PLaMo's multi-token reasoning end marker. - if not self.thinking_enabled: - return input_ids - end_ids = self._reasoning_end_token_id_sequence - for i in range(len(input_ids) - len(end_ids), -1, -1): - if input_ids[i : i + len(end_ids)] == end_ids: - return input_ids[i + len(end_ids) :] - return input_ids - - def _strip_unfinished_marker(self, value: str | None) -> str | None: - if not value: - return value - marker_start = value.rfind("<|plamo:") - if marker_start < 0: - return value or None - suffix = value[marker_start:] - # Strip a trailing partial marker at the token-ID level. - suffix_ids = tuple( - self.model_tokenizer.encode(suffix, add_special_tokens=False) - ) - if any( - len(seq) > len(suffix_ids) and seq[: len(suffix_ids)] == suffix_ids - for seq in self._terminal_token_id_sequences.values() - ): - return value[:marker_start] or None - return value or None - - def _coalesce_finished_tool_events( - self, events: list[SemanticEvent] - ) -> list[SemanticEvent]: - combined = [] - for (kind, _), group in groupby( - events, key=lambda event: (event.type, event.tool_index) - ): - chunks = list(group) - if kind not in (EventType.TOOL_NAME, EventType.ARG_VALUE_CHUNK): - combined.extend(chunks) - continue - if kind == EventType.TOOL_NAME and chunks[0].tool_index < 0: - continue - value = "".join(chunk.value for chunk in chunks) - if kind == EventType.TOOL_NAME: - value = self._strip_unfinished_marker(value) or "" - if value: - combined.append(replace(chunks[0], value=value)) - return combined - - def _events_to_delta( - self, - events: list[SemanticEvent], - finished: bool = False, - ) -> DeltaMessage | None: - # Coalesce flushed tool fragments before stripping truncated markers. - if not finished: - return super()._events_to_delta(events) - - events = self._coalesce_finished_tool_events(events) - delta = super()._events_to_delta(events, finished=True) - if delta is None: - return None - - delta.reasoning = self._strip_unfinished_marker(delta.reasoning) - if not self.skip_tool_parsing: - delta.content = self._strip_unfinished_marker(delta.content) - # The flush emits the remaining (never-streamed) tool-call - # arguments; an empty remainder must match non-streaming - # extraction (_extract_args_json), which normalizes it to '{}'. - for tc in delta.tool_calls or []: - if tc.function is not None and not (tc.function.arguments or "").strip(): - tc.function.arguments = "{}" - if delta.reasoning is None and delta.content is None and not delta.tool_calls: - return None - return delta - - def _handle_arg_chunk( - self, - event: SemanticEvent, - deltas: list[DeltaToolCall], - ) -> None: - # Strip truncated marker suffixes and preserve the first argument delta. - idx = event.tool_index - slot = self._tool_slots[idx] - if (marker_pos := event.value.rfind("<|plamo:")) >= 0: - stripped = event.value[:marker_pos] - try: - json.loads(slot.args + stripped) - except ValueError: - pass - else: - event = replace(event, value=stripped) - name_sent_before = slot.name_sent - super()._handle_arg_chunk(event, deltas) - if event.value and not name_sent_before and slot.name_sent: - deltas.append( - DeltaToolCall( - index=idx, - function=DeltaFunctionCall(arguments=event.value), - ) - ) - - def _extract_args_json(self, raw_args: str, func_name: str) -> str: - # Return the raw arguments as JSON - # since PLaMo3 generates function names and arguments separately. - return raw_args.strip() or "{}" - - def extract_reasoning( - self, - model_output: str, - request: ChatCompletionRequest | ResponsesRequest, - ) -> tuple[str | None, str | None]: - reasoning, content = super().extract_reasoning(model_output, request) - return self._strip_unfinished_marker(reasoning), content diff --git a/vllm/parser/seed_oss.py b/vllm/parser/seed_oss.py deleted file mode 100644 index ceaa04f561..0000000000 --- a/vllm/parser/seed_oss.py +++ /dev/null @@ -1,31 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""seed_oss parser for tool calls and reasoning. - -seed_oss shares the Qwen3 XML grammar exactly; only the wrapper token -strings and the turn-boundary tokens differ:: - - -> - -> - -> - -> - <|im_start|> -> - <|im_end|> -> - -```` and ```` are byte-identical, so the -entire transition table and ``_qwen3_arg_converter`` are inherited from -:class:`Qwen3Parser` unchanged. -""" - -from __future__ import annotations - -from vllm.parser.qwen3 import Qwen3Parser - - -class SeedOssParser(Qwen3Parser): - CONFIG_NAME = "seed_oss" - THINK_START = "" - THINK_END = "" - TOOL_START = "" - TOOL_END = "" - TURN_BOUNDARIES = frozenset(("", "")) diff --git a/vllm/parser/step3p5.py b/vllm/parser/step3p5.py deleted file mode 100644 index 65b3212d1a..0000000000 --- a/vllm/parser/step3p5.py +++ /dev/null @@ -1,49 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Step-3.5 / Step-3.7 parser for tool calls and reasoning. - -Both models use the Qwen3 XML tool-call grammar and ```` reasoning -unchanged. Two things differ from :class:`Qwen3Parser`: - -* The chat template always prefills ``\\n`` and has no switch to turn - thinking off, so ``enable_thinking`` is ignored. -* The model closes reasoning with ``\\n\\n``, so trailing reasoning - whitespace is stripped. -""" - -from __future__ import annotations - -import functools -from dataclasses import replace -from typing import TYPE_CHECKING - -from vllm.parser.engine.parser_engine_config import ParserEngineConfig -from vllm.parser.qwen3 import CHATML_TURN_BOUNDARIES, Qwen3Parser, qwen3_config - -if TYPE_CHECKING: - from vllm.tokenizers import TokenizerLike - from vllm.tool_parsers.abstract_tool_parser import Tool - - -@functools.cache -def step3p5_config() -> ParserEngineConfig: - return replace( - qwen3_config( - thinking=True, - name="step3p5", - turn_boundary_tokens=CHATML_TURN_BOUNDARIES, - ), - strip_trailing_reasoning_whitespace=True, - ) - - -class Step3p5Parser(Qwen3Parser): - def __init__( - self, - tokenizer: TokenizerLike, - tools: list[Tool] | None = None, - **kwargs, - ) -> None: - kwargs.setdefault("parser_engine_config", step3p5_config()) - super().__init__(tokenizer, tools, **kwargs) - self.thinking_enabled = True diff --git a/vllm/platforms/__init__.py b/vllm/platforms/__init__.py index c5fe7d8070..ea61a8537a 100644 --- a/vllm/platforms/__init__.py +++ b/vllm/platforms/__init__.py @@ -6,7 +6,6 @@ import traceback from itertools import chain from typing import TYPE_CHECKING -from vllm import envs from vllm.plugins import PLATFORM_PLUGINS_GROUP, load_plugins_by_group from vllm.utils.import_utils import resolve_obj_by_qualname @@ -125,80 +124,13 @@ def _is_amd_zen_cpu() -> bool: return "AuthenticAMD" in cpuinfo and "avx512" in cpuinfo -def cpu_platform_plugin() -> str | None: - logger.debug("Checking if CPU platform is available.") - is_cpu = envs.VLLM_TARGET_DEVICE == "cpu" - if is_cpu: - logger.debug( - "Confirmed CPU platform is available because " - "VLLM_TARGET_DEVICE is set to CPU." - ) - else: - try: - is_cpu = vllm_version_matches_substr("cpu") - if is_cpu: - logger.debug( - "Confirmed CPU platform is available because vLLM is built " - "with CPU." - ) - if not is_cpu: - import sys - - is_cpu = sys.platform.startswith("darwin") - if is_cpu: - logger.debug( - "Confirmed CPU platform is available because the machine " - "is MacOS." - ) - except Exception as e: - logger.debug("CPU platform is not available because: %s", str(e)) - - if not is_cpu: - return None - - if _is_amd_zen_cpu(): - try: - import zentorch # noqa: F401 - - logger.info( - "AMD Zen CPU detected with zentorch installed, using ZenCpuPlatform." - ) - return "vllm.platforms.zen_cpu.ZenCpuPlatform" - except ImportError: - logger.debug( - "AMD Zen CPU detected but zentorch not installed, " - "falling back to CpuPlatform." - ) - except OSError: - # An ABI-mismatched build fails here with an undefined-symbol - # error; other failures are not known to be safe to recover from. - logger.warning( - "AMD Zen CPU detected but zentorch failed to import, falling " - "back to CpuPlatform. This usually means the zentorch build " - "does not match the installed torch version.", - exc_info=True, - ) - - return "vllm.platforms.cpu.CpuPlatform" - - builtin_platform_plugins = { "cuda": cuda_platform_plugin, "rocm": rocm_platform_plugin, - "cpu": cpu_platform_plugin, } def resolve_current_platform_cls_qualname() -> str: - # An explicit CPU target is authoritative. Native CPU-only CI jobs reuse - # an accelerator wheel and can run on accelerator hosts, so probing every - # plugin would otherwise activate both CPU and the host accelerator. - if envs.VLLM_TARGET_DEVICE == "cpu": - cpu_platform_cls_qualname = cpu_platform_plugin() - assert cpu_platform_cls_qualname is not None - logger.debug("Explicitly selected CPU platform.") - return cpu_platform_cls_qualname - platform_plugins = load_plugins_by_group(PLATFORM_PLUGINS_GROUP) activated_plugins = [] diff --git a/vllm/platforms/cpu.py b/vllm/platforms/cpu.py deleted file mode 100644 index 91ff96ab33..0000000000 --- a/vllm/platforms/cpu.py +++ /dev/null @@ -1,797 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -import glob -import os -import platform -import subprocess -import sys -from typing import TYPE_CHECKING - -import torch - -from vllm import envs -from vllm.logger import init_logger -from vllm.utils.cpu_resource_utils import ( - DEVICE_CONTROL_ENV_VAR, - get_memory_node_info, - get_visible_memory_node, -) -from vllm.utils.mem_constants import GiB_bytes -from vllm.v1.attention.backends.mla.prefill.registry import MLAPrefillBackendEnum -from vllm.v1.attention.backends.registry import AttentionBackendEnum - -from .interface import CpuArchEnum, Platform, PlatformEnum - -logger = init_logger(__name__) - -if TYPE_CHECKING: - from vllm.config import VllmConfig - from vllm.v1.attention.backends.mla.prefill.base import ( - MLAPrefillBackend, - ) - from vllm.v1.attention.backends.mla.prefill.selector import MLAPrefillSelectorConfig - from vllm.v1.attention.selector import AttentionSelectorConfig -else: - VllmConfig = None - - -def _cpu_mamba_backend(vllm_config: VllmConfig) -> str: - """Return the CPU state backend selected by the model configuration.""" - model_config = vllm_config.model_config - if model_config is None: - return "none" - - hf_config = getattr(model_config, "hf_text_config", None) - model_type = str(getattr(hf_config, "model_type", "")).lower() - architecture = str(getattr(model_config, "architecture", "")).lower() - layer_types = getattr(hf_config, "layer_types", None) - has_linear_attention = isinstance(layer_types, (list, tuple)) and ( - "linear_attention" in layer_types - ) - - try: - has_inner_state = bool(model_config.has_inner_state) - except (AttributeError, RuntimeError): - has_inner_state = False - - if not (has_inner_state or has_linear_attention): - return "none" - - fallback_backend = model_type or architecture or "unknown" - if not has_linear_attention: - return fallback_backend - - try: - model_cls, _ = model_config.registry.resolve_model_cls( - model_config.architecture, - model_config=model_config, - ) - except Exception: - return fallback_backend - - try: - state_dtypes = model_cls.get_mamba_state_dtype_from_config(vllm_config) - except Exception: - return fallback_backend - - if not isinstance(state_dtypes, tuple) or len(state_dtypes) != 2: - return fallback_backend - - if vllm_config.cache_config.mamba_ssm_cache_dtype == "float16": - cache_dtype = torch.float16 - elif vllm_config.cache_config.mamba_ssm_cache_dtype == "bfloat16": - cache_dtype = torch.bfloat16 - else: - return fallback_backend - - if state_dtypes[1] == cache_dtype: - return "gdn" - - return fallback_backend - - -def get_max_threads(pid=0): - if hasattr(os, "sched_getaffinity"): - return len(os.sched_getaffinity(pid)) - elif platform.system() == "Darwin": - return os.cpu_count() - else: - raise NotImplementedError("Unsupported OS") - - -class CpuPlatform(Platform): - _enum = PlatformEnum.CPU - device_name: str = "cpu" - device_type: str = "cpu" - dispatch_key: str = "CPU" - dist_backend: str = "gloo" - device_control_env_var = DEVICE_CONTROL_ENV_VAR - - @property - def supported_dtypes(self) -> list[torch.dtype]: - if self.get_cpu_architecture() == CpuArchEnum.POWERPC: - return [torch.bfloat16, torch.float32, torch.float16] - elif self.get_cpu_architecture() == CpuArchEnum.ARM and sys.platform.startswith( - "darwin" - ): - # sysctl exits non-zero when the OID is absent, so check_output would - # raise instead of letting the fp16/fp32 fallback below run. - bf16 = ( - subprocess.run( - ["sysctl", "-n", "hw.optional.arm.FEAT_BF16"], capture_output=True - ).stdout.strip() - == b"1" - ) - if bf16: - return [torch.bfloat16, torch.float16, torch.float32] - return [torch.float16, torch.float32] - elif self.get_cpu_architecture() == CpuArchEnum.RISCV: - return [torch.bfloat16, torch.float16, torch.float32] - # x86/aarch64 CPU has supported both bf16 and fp16 natively. - return [torch.bfloat16, torch.float16, torch.float32] - - @classmethod - def check_runner_kv_caches_multi_layer(cls) -> None: - pass - - @classmethod - def get_device_name(cls, device_id: int = 0) -> str: - return "cpu" - - @classmethod - def get_attn_backend_cls( - cls, - selected_backend: "AttentionBackendEnum", - attn_selector_config: "AttentionSelectorConfig", - num_heads: int | None = None, - ) -> str: - if attn_selector_config.use_sparse: - raise NotImplementedError("Sparse Attention is not supported on CPU.") - if attn_selector_config.use_mla: - amx_available = ( - cls.get_cpu_architecture() == CpuArchEnum.X86 - and torch.cpu._is_amx_tile_supported() - ) - if amx_available and selected_backend != AttentionBackendEnum.CPU_MLA: - # Prefer AMX when available, unless CPU_MLA was explicitly requested. - if ( - selected_backend - and selected_backend != AttentionBackendEnum.AMX_MLA - ): - logger.info("Cannot use %s backend on CPU.", selected_backend) - logger.info_once("Using %s backend.", AttentionBackendEnum.AMX_MLA.name) - return AttentionBackendEnum.AMX_MLA.get_path() - # Reference MLA implementation on CPU. Performance is not the - # goal here; the backend simply wires the CPU decode kernel - # (`mla_decode_kvcache`) and an SDPA-based prefill together with - # the shared MLA scaffolding so that DeepSeek-style models can - # execute on CPU. - if selected_backend and selected_backend not in ( - AttentionBackendEnum.CPU_MLA, - AttentionBackendEnum.AMX_MLA, - ): - logger.info("Cannot use %s backend on CPU.", selected_backend) - logger.info_once("Using %s backend.", AttentionBackendEnum.CPU_MLA.name) - return AttentionBackendEnum.CPU_MLA.get_path() - if selected_backend and selected_backend != AttentionBackendEnum.CPU_ATTN: - logger.info("Cannot use %s backend on CPU.", selected_backend) - return AttentionBackendEnum.CPU_ATTN.get_path() - - @classmethod - def get_mla_prefill_backend_cls( - cls, - mla_selector_config: "MLAPrefillSelectorConfig", - ) -> "type[MLAPrefillBackend]": - """Get the MLA prefill backend class of a device.""" - for backend_enum in (MLAPrefillBackendEnum.ZEN_CPU, MLAPrefillBackendEnum.CPU): - try: - cpu_backend_cls = backend_enum.get_class() - except ImportError: - continue - if cpu_backend_cls.is_available(): - return cpu_backend_cls - raise ValueError("No valid CPU MLA prefill backend found.") - - @classmethod - def get_device_total_memory(cls, device_id: int = 0) -> int: - meminfo = get_memory_node_info(device_id) - - return meminfo.total_memory - - @classmethod - def set_device(cls, device: torch.device) -> None: - """Set the device for the current platform.""" - torch.cpu.set_device(device) - - @classmethod - def manual_seed_all(cls, seed: int) -> None: - pass - - @classmethod - def inference_mode(cls): - return torch.no_grad() - - @classmethod - def check_and_update_config(cls, vllm_config: VllmConfig) -> None: - model_config = vllm_config.model_config - - # Import lazily: vllm.triton_utils imports vllm.platforms.current_platform, - # which is still being resolved while this platform class is loading. - from vllm.triton_utils import HAS_TRITON - - if cls.get_cpu_architecture() == CpuArchEnum.X86 and not HAS_TRITON: - logger.warning_once( - "Triton is not installed. triton-cpu is expected on x86 CPUs." - ) - - cache_config = vllm_config.cache_config - - is_deepseek_v4 = ( - model_config is not None - and getattr(model_config.hf_config, "model_type", None) == "deepseek_v4" - ) - - # The CPU MLA decode kernel only compiles with block_size=16 today - # (see csrc/cpu/mla_decode.cpp). If the model uses MLA we override - # the default block size regardless of user preference to avoid a - # runtime kernel dispatch failure. AMX MLA has no such constraint - # (same AMX-available condition as get_attn_backend_cls), so it's - # excluded from this override. DeepSeek-V4 is also excluded here and - # handled in its own branch below: its sparse-MLA and indexer - # backends declare block_size=256 as their only supported kernel - # block size (DeepseekV4SparseMLABackend/DeepseekV4IndexerBackend), - # so it needs the same override-regardless-of-preference treatment - # as CPU MLA, just with a different value. - cpu_mla_enabled = ( - not is_deepseek_v4 - and model_config is not None - and getattr(model_config, "use_mla", False) - ) - amx_mla_enabled = ( - cpu_mla_enabled - and cls.get_cpu_architecture() == CpuArchEnum.X86 - and torch.cpu._is_amx_tile_supported() - and vllm_config.attention_config.backend != AttentionBackendEnum.CPU_MLA - ) - reference_cpu_mla_enabled = cpu_mla_enabled and not amx_mla_enabled - # DeepSeek-V4's CPU attention/indexer kernels - # (csrc/cpu/sgl-kernels/{flash_mla,store_cache,compressor, - # paged_mqa_logits,topk}.cpp) are AMX-kernel-backed and built on the - # same paged/position-indexed conventions as the GPU/XPU backends - # (block_table/slot_mapping addressing throughout, chunk metadata - # that already carries per-token causal offsets for partial/extend - # continuation) -- so they support chunked prefill and prefix - # caching the same way. `amx_mla_enabled` above deliberately - # excludes DeepSeek-V4 (it has its own block-size requirements, - # unrelated to chunked-prefill support), so it can't be reused here. - amx_mla_or_dsv4_enabled = amx_mla_enabled or ( - is_deepseek_v4 - and cls.get_cpu_architecture() == CpuArchEnum.X86 - and torch.cpu._is_amx_tile_supported() - ) - if reference_cpu_mla_enabled: - if cache_config.user_specified_block_size and cache_config.block_size != 16: - logger.warning( - "CPU MLA backend requires block_size=16, overriding " - "user-specified block_size=%s.", - cache_config.block_size, - ) - cache_config.block_size = 16 - elif is_deepseek_v4: - if ( - cache_config.user_specified_block_size - and cache_config.block_size != 256 - ): - logger.warning( - "DeepSeek-V4 CPU backend requires block_size=256, " - "overriding user-specified block_size=%s.", - cache_config.block_size, - ) - cache_config.block_size = 256 - elif ( - cache_config.user_specified_block_size and cache_config.block_size % 32 != 0 - ): - logger.warning( - "CPU backend prefers block_size is multiples of 32, " - "otherwise the performance is not optimized." - ) - - # Accelerated GDN uses AMX tiles or AVX-512BF16 VDPBF16PS. - if ( - torch.cpu._is_avx512_bf16_supported() - and cache_config.mamba_ssm_cache_dtype != "float32" - ): - mamba_backend = _cpu_mamba_backend(vllm_config) - if ( - cache_config.mamba_ssm_cache_dtype in ("float16", "bfloat16") - and mamba_backend == "gdn" - ): - logger.info( - "Using %s SSM state storage for the CPU accelerated GDN backend.", - cache_config.mamba_ssm_cache_dtype, - ) - else: - cache_config.mamba_ssm_cache_dtype = "float32" - logger.warning( - "Reset SSM cache type to float32 for accelerated GDN mamba " - "backend '%s'.", - mamba_backend, - ) - - # Legacy setting - env_key = "VLLM_CPU_KVCACHE_SPACE" - if env_key in os.environ and os.environ[env_key] != "": - kv_cache_space = int(os.environ[env_key]) - cache_config.kv_cache_memory_bytes = kv_cache_space * GiB_bytes - - scheduler_config = vllm_config.scheduler_config - # async scheduling is not required on CPU - scheduler_config.async_scheduling = False - - parallel_config = vllm_config.parallel_config - if ( - os.environ.get("VLLM_ENABLE_V1_MULTIPROCESSING", "1") == "1" - and parallel_config.distributed_executor_backend == "uni" - ): - # OMP requires the MP executor to function correctly, UniProc - # is not supported as it is not possible to set the OMP - # environment correctly - parallel_config.distributed_executor_backend = "mp" - - if parallel_config.worker_cls == "auto": - parallel_config.worker_cls = "vllm.v1.worker.cpu_worker.CPUWorker" - # Disable DBO - if parallel_config.enable_dbo: - logger.warning_once("Dual-Batch Overlap is not supported on CPU, disabled.") - parallel_config.enable_dbo = False - - # Note: workaround for v1 gpu_model_runner - from vllm.config import CompilationMode - - vllm_config.compilation_config.cudagraph_capture_sizes = [] - - compilation_config = vllm_config.compilation_config - if vllm_config.compilation_config.mode == CompilationMode.VLLM_COMPILE: - # Note: vLLM V1 is using PIECEWISE level compilation, which will - # take time to compile kernels just-in-time with the inductor - # backend. For CPU CI tests, most of them are executed fast and - # compilations consume too much time, even with torch compile - # cache. So use VLLM_CPU_CI_ENV to indicate the CI environment, - # and just execute model with dynamo + eager mode to save time. - # VLLM_CPU_CI_ENV is only used as an internal variable. - backend = "eager" if envs.VLLM_CPU_CI_ENV else "inductor" - - compilation_config.mode = CompilationMode.DYNAMO_TRACE_ONCE - compilation_config.backend = backend - compilation_config.inductor_compile_config.update( - { - "dce": True, - "size_asserts": False, - "nan_asserts": False, - "epilogue_fusion": True, - "cpp.dynamic_threads": True, - } - ) - compilation_config.ir_enable_torch_wrap = False - - if vllm_config.lora_config is not None: - compilation_config.mode = CompilationMode.NONE - - if ( - cls.get_cpu_architecture() == CpuArchEnum.ARM - and "+gelu" not in compilation_config.custom_ops - and "-gelu" not in compilation_config.custom_ops - ): - compilation_config.custom_ops.append("+gelu") - if ( - cls.get_cpu_architecture() == CpuArchEnum.ARM - and "+gelu_tanh" not in compilation_config.custom_ops - and "-gelu_tanh" not in compilation_config.custom_ops - ): - compilation_config.custom_ops.append("+gelu_tanh") - if ( - cls.get_cpu_architecture() == CpuArchEnum.ARM - and "+gelu_and_mul" not in compilation_config.custom_ops - and "-gelu_and_mul" not in compilation_config.custom_ops - ): - compilation_config.custom_ops.append("+gelu_and_mul") - - vllm_config.profiler_config.torch_profiler_dump_cuda_time_total = False - - assert vllm_config.device_config.device_type == "cpu" - - # - # Environment variables for CPU executor - # - - os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" - - # Note: to avoid the error 'nthreads cannot be larger than environment - # variable "NUMEXPR_MAX_THREADS" (64)'. - os.environ["NUMEXPR_MAX_THREADS"] = str(get_max_threads()) - - # Disable torch async compiling which won't work with daemonic processes - os.environ["TORCHINDUCTOR_COMPILE_THREADS"] = "1" - - # Disable multi-stream for shared experts as no Stream on CPU - os.environ["VLLM_DISABLE_SHARED_EXPERTS_STREAM"] = "1" - - # Avoid inductor generates num_thread() and breaks the thread binding - os.environ["TORCHINDUCTOR_CPP_DYNAMIC_THREADS"] = "1" - - # NIXL's Mamba descriptors require DS conv state storage. Select it - # before cache shapes are created, while preserving an explicit layout. - conv_state_layout_env = "VLLM_SSM_CONV_STATE_LAYOUT" - if conv_state_layout_env not in os.environ: - kv_transfer_config = vllm_config.kv_transfer_config - uses_nixl = kv_transfer_config is not None and any( - kv_transfer_config.has_connector(name) - for name in ( - "NixlConnector", - "NixlPullConnector", - "NixlPushConnector", - ) - ) - if uses_nixl: - os.environ[conv_state_layout_env] = "DS" - elif torch.cpu._is_avx512_bf16_supported(): - os.environ[conv_state_layout_env] = "SD" - - ld_preload_str = os.getenv("LD_PRELOAD", "") - cpu_architecture = Platform.get_cpu_architecture() - - if ( - platform.system() == "Linux" - and cpu_architecture - in ( - CpuArchEnum.ARM, - CpuArchEnum.POWERPC, - CpuArchEnum.X86, - CpuArchEnum.S390X, - ) - and not ( - "libomp" in ld_preload_str - or "libgomp" in ld_preload_str - or "libiomp" in ld_preload_str - ) - ): - # We need to LD_PRELOAD PyTorch's libgomp, otherwise only - # one core will be properly utilized when we thread-bind - # See: https://github.com/vllm-project/vllm/issues/27369 - # TODO: Remove once: - # https://github.com/pytorch/pytorch/issues/166087 is fixed - - # We need to find the location of PyTorch's libgomp - torch_pkg = os.path.dirname(torch.__file__) - site_root = os.path.dirname(torch_pkg) - # Search both torch.libs and torch/lib - See: - # https://github.com/vllm-project/vllm/issues/30470 - torch_libs_paths = [ - os.path.join(site_root, "torch.libs"), - os.path.join(torch_pkg, "lib"), - ] - pytorch_libgomp_so_candidates = [] - for torch_libs in torch_libs_paths: - pytorch_libgomp_so_candidates.extend( - glob.glob(os.path.join(torch_libs, "libgomp*.so*")) - ) - if pytorch_libgomp_so_candidates: - pytorch_libgomp_so = pytorch_libgomp_so_candidates[0] - if ld_preload_str: - ld_preload_str += ":" - ld_preload_str += pytorch_libgomp_so - os.environ["LD_PRELOAD"] = ld_preload_str - - # LD_PRELOAD libtcmalloc, bundled under vllm/libs to reduce - # memory allocation overhead - if ( - platform.system() == "Linux" - and cpu_architecture - in (CpuArchEnum.ARM, CpuArchEnum.X86, CpuArchEnum.S390X) - and "libtcmalloc" not in ld_preload_str - ): - vllm_pkg = os.path.dirname(os.path.dirname(__file__)) - tcmalloc_so = None - for pattern in ("libtcmalloc_minimal*.so*", "libtcmalloc.so*"): - tcmalloc_so_candidates = glob.glob( - os.path.join(vllm_pkg, "libs", pattern) - ) - if tcmalloc_so_candidates: - tcmalloc_so = tcmalloc_so_candidates[0] - break - - if tcmalloc_so is not None: - if ld_preload_str: - ld_preload_str = f"{tcmalloc_so}:{ld_preload_str}" - else: - ld_preload_str = tcmalloc_so - os.environ["LD_PRELOAD"] = ld_preload_str - - os.environ["LOCAL_WORLD_SIZE"] = str( - vllm_config.parallel_config.tensor_parallel_size - ) - - if ( - model_config is not None - and model_config.use_mla - and not amx_mla_or_dsv4_enabled - ): - logger.info_once( - "MLA is enabled on a non-GPU platform; forcing chunked " - "prefill and prefix caching to be disabled." - ) - vllm_config.scheduler_config.enable_chunked_prefill = False - vllm_config.cache_config.enable_prefix_caching = False - vllm_config.scheduler_config.max_num_batched_tokens = max( - vllm_config.model_config.max_model_len, - vllm_config.scheduler_config.DEFAULT_MAX_NUM_BATCHED_TOKENS, - ) - - @classmethod - def update_block_size_for_backend(cls, vllm_config: "VllmConfig") -> None: - model_config = vllm_config.model_config - if model_config is None: - return - backend_classes = cls._find_non_ssm_backends(vllm_config) - if not backend_classes: - return - - # Not in check_and_update_config(), which reruns with every - # VllmConfig.__post_init__ and would undo the hybrid alignment below. - cache_config = vllm_config.cache_config - if not cache_config.user_specified_block_size: - cache_config.block_size = cls._preferred_block_size_for_backends( - backend_classes, 128, vllm_config - ) - if not model_config.is_hybrid: - return - - # reconcile attention and mamba page sizes - pre_block_size = cache_config.block_size - cls._align_hybrid_block_size(vllm_config, backend_classes[0]) - cls._check_aligned_block_size(vllm_config, backend_classes, pre_block_size) - - @classmethod - def discover_numa_topology(cls) -> list[list[int]]: - """Discover NUMA topology and keep the last physical core of each numa - into one core group list for nixl start_kv_load() - """ - SYS_NODE = "/sys/devices/system/node" - SYS_CPU = "/sys/devices/system/cpu" - - if not (os.path.exists(SYS_NODE) and os.path.exists(SYS_CPU)): - return [] - - use_highest_sibling = cls.get_cpu_architecture() == CpuArchEnum.X86 - core_rsv_for_kv = [] - for node in os.listdir(SYS_NODE): - if not node.startswith("node") or not node[4:].isdigit(): - continue - node_path = f"{SYS_NODE}/{node}" - - seen_phys = set() - for cpu in os.listdir(node_path): - if not cpu.startswith("cpu") or not cpu[3:].isdigit(): - continue - - cpu_id = int(cpu[3:]) - # thread_siblings based on cpu_id - path = f"{SYS_CPU}/cpu{cpu_id}/topology/thread_siblings_list" - - if os.path.exists(path): - try: - with open(path) as f: - s = f.read() - cpus: list[int] = [] - for part in s.strip().split(","): - if "-" in part: - a, b = map(int, part.split("-")) - cpus.extend(range(a, b + 1)) - else: - cpus.append(int(part)) - siblings = cpus if cpus else [cpu_id] - except (OSError, ValueError): - siblings = [cpu_id] - else: - siblings = [cpu_id] - - phys = max(siblings) if use_highest_sibling else min(siblings) - - if phys not in seen_phys: - seen_phys.add(phys) - - if len(seen_phys) > 0: - core_rsv_for_kv.append(list(seen_phys)) - - return core_rsv_for_kv - - @classmethod - def is_pin_memory_available(cls) -> bool: - return False - - @classmethod - def get_punica_wrapper(cls) -> str: - return "vllm.lora.punica_wrapper.punica_cpu.PunicaWrapperCPU" - - @classmethod - def get_device_communicator_cls(cls) -> str: - """Get device specific communicator class for distributed communication.""" - return "vllm.distributed.device_communicators.cpu_communicator.CpuCommunicator" # noqa - - @classmethod - def supports_structured_output(cls) -> bool: - return True - - @classmethod - def opaque_attention_op(cls) -> bool: - return True - - @classmethod - def support_hybrid_kv_cache(cls) -> bool: - return True - - @classmethod - def num_compute_units(cls, device_id: int = 0) -> int: - return torch.get_num_threads() - - @classmethod - def import_kernels(cls) -> None: - if Platform.get_cpu_architecture() in (CpuArchEnum.X86,): - # Note: The lib name is _C_AVX2/AVX512, but the module name is _C. - # This will cause a exception "dynamic module does define - # module export function". But the library is imported - # successfully. So ignore the exception for now, until we find - # a solution. - ignored_msg = "dynamic module does not define module export function" - if torch.cpu._is_avx512_supported(): - if torch.cpu._is_avx512_bf16_supported(): - try: - import vllm._C # noqa: F401 - except ImportError as e: - logger.warning_once( - "Failed to import from vllm._C: %s", repr(e) - ) - else: - try: - import vllm._C_AVX512 # noqa: F401 - except ImportError as e: - if ignored_msg not in e.msg: - logger.warning_once( - "Failed to import from vllm._C_AVX512: %s", repr(e) - ) - else: - try: - import vllm._C_AVX2 # noqa: F401 - except ImportError as e: - if ignored_msg not in e.msg: - logger.warning_once( - "Failed to import from vllm._C_AVX2: %s", repr(e) - ) - else: - try: - import vllm._C # noqa: F401 - except ImportError as e: - logger.warning_once("Failed to import from vllm._C: %s", repr(e)) - - @classmethod - def register_triton_kernel_overrides(cls) -> None: - """Fallback C++ implementations for core Triton kernels. - - Used when the Triton-CPU backend is unavailable. When Triton-CPU is - present, the Triton kernels can run natively and the overrides - would only shadow them, so registration is skipped. - """ - from vllm.triton_utils import HAS_TRITON - - if HAS_TRITON: - return - - from vllm.triton_utils.dispatcher import register_kernels - from vllm.utils import cpu_triton_utils as cpu_tl - - register_kernels( - { - "vllm.v1.worker.block_table.ComputeSlotMappingKernel.kernel": ( - cpu_tl._compute_slot_mapping_kernel_impl - ), - "vllm.v1.spec_decode.utils.eagle_step_slot_mapping_metadata_kernel": ( - cpu_tl._eagle_step_slot_mapping_metadata_kernel_impl - ), - "vllm.v1.spec_decode.utils.eagle_prepare_inputs_padded_kernel": ( - cpu_tl._eagle_prepare_inputs_padded_kernel_impl - ), - "vllm.v1.spec_decode.utils.eagle_prepare_next_token_padded_kernel": ( - cpu_tl._eagle_prepare_next_token_padded_kernel_impl - ), - "vllm.v1.spec_decode.utils.copy_and_expand_eagle_inputs_kernel": ( - cpu_tl._copy_and_expand_eagle_inputs_kernel_impl - ), - "vllm.v1.spec_decode.utils.copy_and_expand_dflash_inputs_kernel": ( - cpu_tl._copy_and_expand_dflash_inputs_kernel_impl - ), - "vllm.v1.sample.rejection_sampler.rejection_greedy_sample_kernel": ( - cpu_tl._rejection_greedy_sample_kernel_impl - ), - "vllm.v1.sample.rejection_sampler.rejection_random_sample_kernel": ( - cpu_tl._rejection_random_sample_kernel_impl - ), - "vllm.v1.sample.rejection_sampler.expand_kernel": ( - cpu_tl._expand_kernel_impl - ), - "vllm.v1.sample.rejection_sampler.sample_recovered_tokens_kernel": ( - cpu_tl._sample_recovered_tokens_kernel_impl - ), - "vllm.v1.worker.mamba_utils.batch_memcpy_kernel": ( - cpu_tl._batch_memcpy_impl - ), - } - ) - - @classmethod - def pack_kv_cache( - cls, - kv_cache: torch.Tensor, - indices: torch.Tensor, - ) -> None: - """Rewrite the kv cache shape for the current platform.""" - # Import lazily: cpu_attn pulls in _custom_ops, which needs a fully - # initialized vllm.platforms (avoid circular import while CpuPlatform loads). - from vllm._custom_ops import cpu_attn_reshape_and_cache - from vllm.v1.attention.backends.cpu_attn import _get_attn_isa - - # MLA uses a single latent cache of shape [N, block_size, head_size], - # so the classic key/value split does not apply. The MLA backend - # writes the cache itself via `concat_and_cache_mla` inside - # `do_kv_cache_update`, so there is nothing to pack here. - if kv_cache.dim() == 3: - return - - num_blocks, num_kv_heads, block_size, fused_head_size = kv_cache.shape - head_size = fused_head_size // 2 - - # Fused path used by heterogeneous NIXL CPU_ATTN post-processing. - blocks_to_update = kv_cache.index_select(0, indices) - key = blocks_to_update[..., :head_size] - value = blocks_to_update[..., head_size:] - - key_cache, value_cache = kv_cache.view( - num_blocks, num_kv_heads, block_size * 2, head_size - ).chunk(2, dim=2) - - dtype = key.dtype - # For CPU_ATTN, the shape is [N, num_kv_heads, block_size, head_size] - key = key.permute(0, 2, 1, 3).flatten(0, 1) - value = value.permute(0, 2, 1, 3).flatten(0, 1) - - isa = _get_attn_isa(dtype, block_size, head_size) - block_offsets = torch.arange(block_size, device="cpu", dtype=torch.long) - num_blocks = indices.numel() - slot_mapping = ( - block_offsets.reshape(1, block_size) - + indices.reshape(num_blocks, 1) * block_size - ).flatten() - if key_cache.dtype == torch.uint8: - raise NotImplementedError( - "FP8 KV cache is not yet supported with KV transfer on CPU" - ) - cpu_attn_reshape_and_cache( - key, - value, - key_cache, - value_cache, - slot_mapping, - isa, - ) - - @classmethod - def get_current_memory_usage( - cls, device: torch.types.Device | None = None - ) -> float: - allowed_mem_node_list = get_visible_memory_node() - mem_status_list = [get_memory_node_info(i) for i in allowed_mem_node_list] - memory_usage = 0 - for s in mem_status_list: - memory_usage += s.total_memory - s.available_memory - - return memory_usage diff --git a/vllm/platforms/zen_cpu.py b/vllm/platforms/zen_cpu.py deleted file mode 100644 index 1ee45be385..0000000000 --- a/vllm/platforms/zen_cpu.py +++ /dev/null @@ -1,34 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -import torch - -from vllm.logger import init_logger -from vllm.platforms.cpu import CpuPlatform - -logger = init_logger(__name__) - - -class ZenCpuPlatform(CpuPlatform): - """CPU platform with AMD Zen (ZenDNN/zentorch) optimizations. - - Model-load time (dispatch_cpu_unquantized_gemm in layers/utils.py): - - Routes linear ops to zentorch_linear_unary. - - When VLLM_ZENTORCH_WEIGHT_PREPACK=1 (default), eagerly prepacks - weights via zentorch_weight_prepack_for_linear. - """ - - device_name: str = "cpu" - device_type: str = "cpu" - - def is_zen_cpu(self) -> bool: - # is_cpu() also returns True for this platform (inherited from CpuPlatform). - return True - - # fp16 compute needs AVX512-FP16, which AMD ships from Venice (6th Gen - # EPYC) onwards; Turin (5th Gen EPYC) and earlier have no fp16 kernels. - @property - def supported_dtypes(self) -> list[torch.dtype]: - if torch.ops.zentorch.zentorch_is_fp16_supported(): - return [torch.bfloat16, torch.float32, torch.float16] - return [torch.bfloat16, torch.float32] diff --git a/vllm/reasoning/AGENTS.md b/vllm/reasoning/AGENTS.md deleted file mode 100644 index 1f7c8b3557..0000000000 --- a/vllm/reasoning/AGENTS.md +++ /dev/null @@ -1,54 +0,0 @@ -# vllm/reasoning - -Every `--reasoning-parser` name resolves here. The parser has two consumers: -`ParserManager.get_parser` composes it with the tool parser into the unified -`Parser` the serving layer uses, and the structured-output manager -(`vllm/v1/structured_output/__init__.py`) instantiates it directly and calls -`is_reasoning_end` / `is_reasoning_end_streaming` every decode step to decide -when the grammar bitmask starts applying. The `ReasoningParser` interface -itself is legacy: it is kept so existing in-tree parsers and out-of-tree -plugins keep working, not as the base for new ones. - -## Layout - -- `abs_reasoning_parsers.py` — `ReasoningParser` (`extract_reasoning`, - `extract_reasoning_streaming`, `is_reasoning_end`, - `is_reasoning_end_streaming`, `extract_content_ids`, `adjust_request`) and - `ReasoningParserManager`. -- `__init__.py` — `_REASONING_PARSERS_TO_REGISTER`, the name → (file, class) - table that lazily registers every built-in parser. -- `basic_parsers.py` — `BaseThinkingReasoningParser`, the base for parsers - that only need a start/end token pair. -- `_reasoning_parser.py` — one file per family. Either a subclass of a - `ParserReasoningAdapter` generated from a `vllm/parser/.py` engine - grammar (`mistral_reasoning_parser.py`, `kimi_k2_reasoning_parser.py`; the - `_engine_` suffix is not applied consistently), or a self-contained - `ReasoningParser` subclass. - -## Guidelines - -- Do not change the `ReasoningParser` interface in `abs_reasoning_parsers.py`. - Out-of-tree plugins loaded through - `ReasoningParserManager.import_reasoning_parser` subclass it. New - capabilities go on `Parser` in `vllm/parser/`; if a legacy parser needs the - same capability, `DelegatingParser` supplies the default. -- For a new model, implement a unified `Parser` in `vllm/parser/` first, then - expose it here as a thin adapter subclass plus an entry in - `_REASONING_PARSERS_TO_REGISTER`. -- The `_REASONING_PARSERS_TO_REGISTER` key is what users pass on the command - line; treat renames as breaking. -- Keep adapters thin: registration, `adjust_request` tweaks, and class - attributes. Parsing logic belongs in `vllm/parser/`. -- Keep a model-specific fix inside that model's `_reasoning_parser.py`, - or its grammar in `vllm/parser/.py`, where possible. Touch - `abs_reasoning_parsers.py` or `basic_parsers.py` only for behavior every - parser needs, and say so in the PR. -- `is_reasoning_end_streaming` runs on the engine core for every decode step - of every structured-output request. Work on `delta_ids`; re-scanning the - full sequence makes the step cost grow with sequence length. -- The structured-output gate and the frontend's extraction must agree on - where reasoning ends, whether the turn continues with content or a tool - call. Test the end-detection methods with the same inputs as extraction. -- Comment only where the code alone would leave a reader with a question — - a request tweak in `adjust_request`, a marker handled unusually, a model - quirk being worked around. Keep it minimal and concise. diff --git a/vllm/reasoning/basic_parsers.py b/vllm/reasoning/basic_parsers.py deleted file mode 100644 index e1d3092506..0000000000 --- a/vllm/reasoning/basic_parsers.py +++ /dev/null @@ -1,196 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from abc import abstractmethod -from collections.abc import Iterable, Sequence -from itertools import islice -from typing import TYPE_CHECKING - -from vllm.entrypoints.generate.base.protocol import DeltaMessage -from vllm.reasoning.abs_reasoning_parsers import ReasoningParser -from vllm.tokenizers import TokenizerLike - -if TYPE_CHECKING: - from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest - from vllm.entrypoints.openai.responses.protocol import ResponsesRequest - - -class BaseThinkingReasoningParser(ReasoningParser): - """Base class for reasoning parsers that use thinking tokens. - - This class provides common functionality for parsers that use start and end - tokens to delimit reasoning content ( - e.g., ..., ...). - - Subclasses must implement the start and end tokens via abstract - properties. - """ - - @property - @abstractmethod - def start_token(self) -> str: - """The token that starts reasoning content.""" - raise NotImplementedError - - @property - @abstractmethod - def end_token(self) -> str: - """The token that ends reasoning content.""" - raise NotImplementedError - - @property - def reasoning_start_str(self) -> str: - return self.start_token - - @property - def reasoning_end_str(self) -> str: - return self.end_token - - def __init__(self, tokenizer: TokenizerLike, *args, **kwargs): - super().__init__(tokenizer, *args, **kwargs) - - if not self.model_tokenizer: - raise ValueError( - "The model tokenizer must be passed to the ReasoningParser " - "constructor during construction." - ) - - if not self.start_token or not self.end_token: - raise ValueError("start_token and end_token must be defined in subclasses") - - start_token_id = self.vocab.get(self.start_token) - end_token_id = self.vocab.get(self.end_token) - if start_token_id is None or end_token_id is None: - raise RuntimeError( - f"{self.__class__.__name__} reasoning parser could not locate " - "think start/end tokens in the tokenizer!" - ) - self.start_token_id: int = start_token_id - self.end_token_id: int = end_token_id - - def is_reasoning_end(self, input_ids: Sequence[int]) -> bool: - start_token_id = self.start_token_id - end_token_id = self.end_token_id - - for i in range(len(input_ids) - 1, -1, -1): - if input_ids[i] == start_token_id: - return False - if input_ids[i] == end_token_id: - return True - return False - - def is_reasoning_end_streaming( - self, input_ids: Sequence[int], delta_ids: Iterable[int] - ) -> bool: - end_token_id = self.end_token_id - return end_token_id in delta_ids - - def extract_content_ids(self, input_ids: list[int]) -> list[int]: - """Extract the content after the end tokens.""" - if self.end_token_id not in islice(input_ids, 0, max(0, len(input_ids) - 1)): - return [] - else: - return input_ids[input_ids.index(self.end_token_id) + 1 :] - - def extract_reasoning_streaming( - self, - previous_text: str, - current_text: str, - delta_text: str, - previous_token_ids: Sequence[int], - current_token_ids: Sequence[int], - delta_token_ids: Sequence[int], - ) -> DeltaMessage | None: - """Extract reasoning content from a delta message. - Handles streaming output where previous + delta = current. - Uses token IDs for faster processing. - """ - # Skip single special tokens - if len(delta_token_ids) == 1 and ( - delta_token_ids[0] in [self.start_token_id, self.end_token_id] - ): - return None - - # Check if start token is present in previous or delta. - # Keep compatibility with models that don't generate start tokens. - if self.start_token_id in previous_token_ids: - if self.end_token_id in delta_token_ids: - # start token in previous, end token in delta, - # extract reasoning content - end_index = delta_text.find(self.end_token) - reasoning = delta_text[:end_index] - content = delta_text[end_index + len(self.end_token) :] - return DeltaMessage( - reasoning=reasoning, content=content if content else None - ) - elif self.end_token_id in previous_token_ids: - # start token in previous, end token in previous, - # reasoning content continues - return DeltaMessage(content=delta_text) - else: - # start token in previous, no end token in previous or delta, - # reasoning content continues - return DeltaMessage(reasoning=delta_text) - elif self.start_token_id in delta_token_ids: - if self.end_token_id in delta_token_ids: - # start token in delta, end token in delta, - # extract reasoning content - start_index = delta_text.find(self.start_token) - end_index = delta_text.find(self.end_token) - reasoning = delta_text[start_index + len(self.start_token) : end_index] - content = delta_text[end_index + len(self.end_token) :] - return DeltaMessage( - reasoning=reasoning, content=content if content else None - ) - else: - # start token in delta, no end token in delta, - # reasoning content continues - return DeltaMessage(reasoning=delta_text) - else: - # not find thinking start token - return DeltaMessage(content=delta_text) - - def extract_reasoning( - self, model_output: str, request: "ChatCompletionRequest | ResponsesRequest" - ) -> tuple[str | None, str | None]: - """Extract reasoning content from the model output. - - This is the base implementation that works for most models. - Subclasses can override this method for specific behavior. - """ - # Check if the start token is present in the model output, remove it - # if it is present. - model_output_parts = model_output.partition(self.start_token) - model_output = ( - model_output_parts[2] if model_output_parts[1] else model_output_parts[0] - ) - - # For models that may not generate start token, - # assume the reasoning content is always at the start. - if self.end_token not in model_output: - return model_output, None - else: - reasoning, _, content = model_output.partition(self.end_token) - # If generation stops right after end-of-think, return null content - final_content = content or None - return reasoning, final_content - - def count_reasoning_tokens(self, token_ids: Sequence[int]) -> int: - """Count tokens that fall within start/end thinking markers. - - Uses a depth counter so nested spans are handled safely and stray end - tokens do not drive the counter negative. - """ - count = 0 - depth = 0 - for token_id in token_ids: - if token_id == self.start_token_id: - depth += 1 - continue - if token_id == self.end_token_id: - if depth > 0: - depth -= 1 - continue - if depth > 0: - count += 1 - return count diff --git a/vllm/reasoning/beam_reasoning_parser.py b/vllm/reasoning/beam_reasoning_parser.py deleted file mode 100644 index b5d87170f0..0000000000 --- a/vllm/reasoning/beam_reasoning_parser.py +++ /dev/null @@ -1,6 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from vllm.parser.beam import BeamReasoningParser - -__all__ = ["BeamReasoningParser"] diff --git a/vllm/reasoning/cohere_command_reasoning_parser.py b/vllm/reasoning/cohere_command_reasoning_parser.py deleted file mode 100644 index d02091e600..0000000000 --- a/vllm/reasoning/cohere_command_reasoning_parser.py +++ /dev/null @@ -1,60 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from __future__ import annotations - -from collections.abc import Sequence -from typing import TYPE_CHECKING, ClassVar - -from vllm.reasoning import ReasoningParser - -if TYPE_CHECKING: - from vllm.tokenizers import TokenizerLike - - -class BaseCohereCommandReasoningParser(ReasoningParser): - """Expose is_reasoning_end for the structured-output engine. - Everything else runs in - :class:`vllm.parser.cohere_command.CohereCommandParser`. - """ - - melody_preset: ClassVar[str] - - def __init__(self, tokenizer: TokenizerLike, *args, **kwargs): - super().__init__(tokenizer, *args, **kwargs) - self.start_token_id = tokenizer.convert_tokens_to_ids("<|START_THINKING|>") - self.end_token_id = tokenizer.convert_tokens_to_ids("<|END_THINKING|>") - self.chatbot_token_id = tokenizer.convert_tokens_to_ids("<|CHATBOT_TOKEN|>") - - @property - def reasoning_start_str(self) -> str | None: - return "<|START_THINKING|>" - - @property - def reasoning_end_str(self) -> str | None: - return "<|END_THINKING|>" - - def is_reasoning_end(self, input_ids: Sequence[int]) -> bool: - chatbot = self.chatbot_token_id - start = self.start_token_id - end = self.end_token_id - has_end_token = False - - for i in reversed(range(len(input_ids))): - tid = input_ids[i] - if tid == start: - return has_end_token - if tid == chatbot: - return False - if tid == end: - has_end_token = True - - return has_end_token - - -class CohereCommand3ReasoningParser(BaseCohereCommandReasoningParser): - melody_preset = "cmd3" - - -class CohereCommand4ReasoningParser(BaseCohereCommandReasoningParser): - melody_preset = "cmd4" diff --git a/vllm/reasoning/deepseek_r1_reasoning_parser.py b/vllm/reasoning/deepseek_r1_reasoning_parser.py deleted file mode 100644 index 991cf01659..0000000000 --- a/vllm/reasoning/deepseek_r1_reasoning_parser.py +++ /dev/null @@ -1,66 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from collections.abc import Sequence - -from vllm.entrypoints.generate.base.protocol import DeltaMessage -from vllm.reasoning.basic_parsers import BaseThinkingReasoningParser - - -class DeepSeekR1ReasoningParser(BaseThinkingReasoningParser): - """Reasoning parser for DeepSeek R1 model. - - The DeepSeek R1 model uses ... tokens to denote reasoning - text. This parser extracts the reasoning content from the model output. - """ - - @property - def start_token(self) -> str: - """The token that starts reasoning content.""" - return "" - - @property - def end_token(self) -> str: - """The token that ends reasoning content.""" - return "" - - def extract_reasoning_streaming( - self, - previous_text: str, - current_text: str, - delta_text: str, - previous_token_ids: Sequence[int], - current_token_ids: Sequence[int], - delta_token_ids: Sequence[int], - ) -> DeltaMessage | None: - ret = super().extract_reasoning_streaming( - previous_text, - current_text, - delta_text, - previous_token_ids, - current_token_ids, - delta_token_ids, - ) - if ( - ret is not None - and self.start_token_id not in previous_token_ids - and self.start_token_id not in delta_token_ids - ): - if self.end_token_id in delta_token_ids: - # end token in delta with more tokens, - # extract reasoning content and content - end_index = delta_text.find(self.end_token) - reasoning = delta_text[:end_index] - content = delta_text[end_index + len(self.end_token) :] - return DeltaMessage( - reasoning=reasoning, - content=content if content else None, - ) - elif self.end_token_id in previous_token_ids: - # end token in previous, thinking content ends - return DeltaMessage(content=delta_text) - else: - # no end token in previous or delta, reasoning content continues - return DeltaMessage(reasoning=delta_text) - - return ret diff --git a/vllm/reasoning/deepseek_v3_reasoning_parser.py b/vllm/reasoning/deepseek_v3_reasoning_parser.py deleted file mode 100644 index a7613112f4..0000000000 --- a/vllm/reasoning/deepseek_v3_reasoning_parser.py +++ /dev/null @@ -1,96 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from collections.abc import Iterable, Sequence -from typing import TYPE_CHECKING - -from transformers import PreTrainedTokenizerBase - -from vllm.reasoning import ReasoningParser -from vllm.reasoning.deepseek_r1_reasoning_parser import DeepSeekR1ReasoningParser - -from .identity_reasoning_parser import IdentityReasoningParser - -if TYPE_CHECKING: - from vllm.entrypoints.generate.base.protocol import DeltaMessage - from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest - from vllm.entrypoints.openai.responses.protocol import ResponsesRequest - - -class DeepSeekV3ReasoningParser(ReasoningParser): - """V3 parser that delegates to either DeepSeekR1ReasoningParser or - IdentityReasoningParser based on `thinking` and `separate_reasoning`. - """ - - def __init__(self, tokenizer: PreTrainedTokenizerBase, *args, **kwargs): - super().__init__(tokenizer, *args, **kwargs) - - chat_kwargs = kwargs.get("chat_template_kwargs", {}) or {} - thinking = bool(chat_kwargs.get("thinking", False)) - enable_thinking = bool(chat_kwargs.get("enable_thinking", False)) - thinking = thinking or enable_thinking - - self._parser: ReasoningParser - if thinking: - self._parser = DeepSeekR1ReasoningParser(tokenizer, *args, **kwargs) - else: - self._parser = IdentityReasoningParser(tokenizer, *args, **kwargs) - - @property - def reasoning_start_str(self) -> str | None: - return self._parser.reasoning_start_str - - @property - def reasoning_end_str(self) -> str | None: - return self._parser.reasoning_end_str - - def is_reasoning_end(self, input_ids: Sequence[int]) -> bool: - return self._parser.is_reasoning_end(input_ids) - - def is_reasoning_end_streaming( - self, input_ids: Sequence[int], delta_ids: Iterable[int] - ) -> bool: - return self._parser.is_reasoning_end_streaming(input_ids, delta_ids) - - def extract_content_ids(self, input_ids: list[int]) -> list[int]: - return self._parser.extract_content_ids(input_ids) - - def count_reasoning_tokens(self, token_ids: Sequence[int]) -> int: - return self._parser.count_reasoning_tokens(token_ids) - - def extract_reasoning( - self, model_output: str, request: "ChatCompletionRequest | ResponsesRequest" - ) -> tuple[str | None, str | None]: - return self._parser.extract_reasoning(model_output, request) - - def extract_reasoning_streaming( - self, - previous_text: str, - current_text: str, - delta_text: str, - previous_token_ids: Sequence[int], - current_token_ids: Sequence[int], - delta_token_ids: Sequence[int], - ) -> "DeltaMessage | None": - return self._parser.extract_reasoning_streaming( - previous_text, - current_text, - delta_text, - previous_token_ids, - current_token_ids, - delta_token_ids, - ) - - -class DeepSeekV3ReasoningWithThinkingParser(DeepSeekV3ReasoningParser): - """DeepSeekV3ReasoningParser that defaults to thinking mode.""" - - def __init__(self, tokenizer: PreTrainedTokenizerBase, *args, **kwargs): - chat_kwargs = kwargs.get("chat_template_kwargs", {}) or {} - thinking = chat_kwargs.get("thinking", None) - enable_thinking = chat_kwargs.get("enable_thinking", None) - if thinking is None and enable_thinking is None: - chat_kwargs["thinking"] = True - chat_kwargs["enable_thinking"] = True - kwargs["chat_template_kwargs"] = chat_kwargs - super().__init__(tokenizer, *args, **kwargs) diff --git a/vllm/reasoning/deepseek_v41_engine_reasoning_parser.py b/vllm/reasoning/deepseek_v41_engine_reasoning_parser.py deleted file mode 100644 index bace0eb4f8..0000000000 --- a/vllm/reasoning/deepseek_v41_engine_reasoning_parser.py +++ /dev/null @@ -1,6 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from vllm.parser.engine.registered_adapters import DeepSeekV41ParserReasoningAdapter - -__all__ = ["DeepSeekV41ParserReasoningAdapter"] diff --git a/vllm/reasoning/deepseek_v4_engine_reasoning_parser.py b/vllm/reasoning/deepseek_v4_engine_reasoning_parser.py deleted file mode 100644 index 6fa6444b35..0000000000 --- a/vllm/reasoning/deepseek_v4_engine_reasoning_parser.py +++ /dev/null @@ -1,6 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from vllm.parser.engine.registered_adapters import DeepSeekV4ParserReasoningAdapter - -__all__ = ["DeepSeekV4ParserReasoningAdapter"] diff --git a/vllm/reasoning/ernie45_reasoning_parser.py b/vllm/reasoning/ernie45_reasoning_parser.py deleted file mode 100644 index 444467b68e..0000000000 --- a/vllm/reasoning/ernie45_reasoning_parser.py +++ /dev/null @@ -1,157 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from collections.abc import Sequence -from typing import TYPE_CHECKING - -from transformers import PreTrainedTokenizerBase - -from vllm.entrypoints.generate.base.protocol import DeltaMessage -from vllm.reasoning.basic_parsers import BaseThinkingReasoningParser - -if TYPE_CHECKING: - from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest - from vllm.entrypoints.openai.responses.protocol import ResponsesRequest - - -class Ernie45ReasoningParser(BaseThinkingReasoningParser): - """Reasoning parser for Ernie45 thinking model. - The Ernie45 thinking model output format is - abc\n\n\n\ndef\n\n - or abc\n\ndef - """ - - response_start_token: str = "" - response_end_token: str = "" - newline_token: str = "<0x0A>" - - @property - def start_token(self) -> str: - """The token that starts reasoning content.""" - return "" - - @property - def end_token(self) -> str: - """The token that ends reasoning content.""" - return "" - - def __init__(self, tokenizer: PreTrainedTokenizerBase, *args, **kwargs): - super().__init__(tokenizer, *args, **kwargs) - - if not self.model_tokenizer: - raise ValueError( - "The model tokenizer must be passed to the ReasoningParser " - "constructor during construction." - ) - - self.response_start_token_id = self.vocab.get(self.response_start_token) - self.response_end_token_id = self.vocab.get(self.response_end_token) - self.newline_token_id = self.vocab.get(self.newline_token) - - self.parser_token_ids = [self.end_token_id, self.response_end_token_id] - - def extract_reasoning_streaming( - self, - previous_text: str, - current_text: str, - delta_text: str, - previous_token_ids: Sequence[int], - current_token_ids: Sequence[int], - delta_token_ids: Sequence[int], - ) -> DeltaMessage | None: - """Extract reasoning content from a delta message. - Handles streaming output where previous + delta = current. - Uses token IDs for faster processing. - The Ernie45 thinking model output format is - abc\n\n\n\ndef\n\n - or abc\n\ndef - - 'abc' goes to reasoning - - 'def' goes to content - """ - # Skip single special tokens - if len(delta_token_ids) == 1 and ( - delta_token_ids[0] - in [ - self.start_token_id, - self.end_token_id, - self.response_start_token_id, - self.response_end_token_id, - ] - ): - return None - - # No in previous or delta, also need to check for . - # Because the model may have generated without - if self.end_token_id in delta_token_ids: - # in delta with more tokens, - # extract reasoning content and content - think_end_index = delta_text.find(self.end_token) - reasoning = delta_text[:think_end_index] - content = delta_text[think_end_index + len(self.end_token) :] - content = content.lstrip("\n") - response_start_idx = content.find(self.response_start_token) - response_end_idx = content.rfind(self.response_end_token) - if response_start_idx != -1: - content = content[response_start_idx + len(self.response_start_token) :] - if response_end_idx != -1: - content = content[:response_end_idx] - return DeltaMessage( - reasoning=reasoning, - content=content if content else None, - ) - elif self.end_token_id in previous_token_ids: - # in previous, thinking content ends - content = delta_text - if self.response_start_token_id in delta_token_ids: - content = content.lstrip("\n") - response_start_idx = content.find(self.response_start_token) - content = content[response_start_idx + len(self.response_start_token) :] - # if have , remove it - response_end_idx = content.rfind(self.response_end_token) - if response_end_idx != -1: - content = content[:response_end_idx] - elif self.response_end_token_id in delta_token_ids: - response_end_idx = content.rfind(self.response_end_token) - if response_end_idx != -1: - content = content[:response_end_idx] - # remove \n after or - if previous_token_ids[-1] in self.parser_token_ids and ( - len(delta_token_ids) > 0 and delta_token_ids[0] == self.newline_token_id - ): - content = content.lstrip("\n") - # remove \n after \n - if ( - len(previous_token_ids) > 1 - and previous_token_ids[-2] == self.end_token_id - ) and ( - len(delta_token_ids) > 0 and delta_token_ids[0] == self.newline_token_id - ): - content = content.lstrip("\n") - - return DeltaMessage(content=content if content else None) - else: - # no in previous or delta, reasoning content continues - return DeltaMessage(reasoning=delta_text) - - def extract_reasoning( - self, model_output: str, request: "ChatCompletionRequest | ResponsesRequest" - ) -> tuple[str | None, str | None]: - """Extract reasoning content from the model output. - The Ernie45 thinking model output format is - abc\n\n\n\n\ndef\n\n - or abc\n\ndef - - 'abc' goes to reasoning - - 'def' goes to content - Returns: - tuple[Optional[str], Optional[str]]: reasoning content and content - """ - reasoning, content = super().extract_reasoning(model_output, request) - if content: - start_idx = content.find(self.response_start_token) - end_idx = content.rfind(self.response_end_token) - # Simultaneously existing and in the correct order - if start_idx != -1 and end_idx != -1 and start_idx < end_idx: - content = content[start_idx + len(self.response_start_token) : end_idx] - final_content = content or None - - return reasoning, final_content diff --git a/vllm/reasoning/gemma4_engine_reasoning_parser.py b/vllm/reasoning/gemma4_engine_reasoning_parser.py deleted file mode 100644 index e9bc46e9bf..0000000000 --- a/vllm/reasoning/gemma4_engine_reasoning_parser.py +++ /dev/null @@ -1,6 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from vllm.parser.engine.registered_adapters import Gemma4ParserReasoningAdapter - -__all__ = ["Gemma4ParserReasoningAdapter"] diff --git a/vllm/reasoning/gemma4_utils.py b/vllm/reasoning/gemma4_utils.py deleted file mode 100644 index 9566b8af56..0000000000 --- a/vllm/reasoning/gemma4_utils.py +++ /dev/null @@ -1,131 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -# Copyright 2025 Google Inc. HuggingFace Inc. team. All rights reserved. - -"""Gemma4 thinking/reasoning output parsing utilities for offline inference. - -Standalone functions that parse decoded model text to extract structured -thinking content from Gemma4 models. These are pure-Python utilities with -zero heavy dependencies — they work on raw decoded strings from any -inference backend (vLLM, HuggingFace, TGI, etc.). - -For the OpenAI-compatible API reasoning parser (streaming + -non-streaming), see ``vllm.reasoning.gemma4_reasoning_parser``. -For tool call parsing, see ``vllm.tool_parsers.gemma4_utils``. - -Usage with vLLM offline inference:: - - from vllm import LLM, SamplingParams - from vllm.reasoning.gemma4_utils import parse_thinking_output - - llm = LLM(model="google/gemma-4-it") - outputs = llm.generate(prompt, SamplingParams(...)) - text = tokenizer.decode(outputs[0].outputs[0].token_ids, skip_special_tokens=False) - - # Extract thinking / answer (works with or without enable_thinking) - result = parse_thinking_output(text) - print(result["thinking"]) # chain-of-thought or None - print(result["answer"]) # final answer - -Ported from ``transformers.models.gemma4.utils_gemma4`` so that vLLM users -do not need a transformers dependency for output parsing. -""" - -# ---- Thinking Mode Utility ---- - -# Thinking delimiter tokens as they appear in decoded text. -# Gemma4 uses <|channel> (start) and (end) as thinking delimiters. -_THINKING_START_TAG = "<|channel>" -_THINKING_END_TAG = "" - -# Sentinel tokens that may appear in decoded output. -_TURN_END_TAG = "" - - -def parse_thinking_output(text: str) -> dict[str, str | None]: - """Parse decoded Gemma4 model output. - - Use this on **all** Gemma4 output regardless of whether thinking mode - was enabled. It handles three cases: - - 1. **Thinking enabled, tags present** — splits on ``<|channel>``/ - ```` to separate chain-of-thought from the answer and - strips the ``thought\\n`` role label. - 2. **Thinking disabled, spurious label** — strips the bare - ``thought\\n`` prefix that some Gemma4 models emit even - without thinking mode. - 3. **Clean output** — returns the text unchanged. - - The answer text is always cleaned of trailing sentinel tokens - (````, ````, etc.). - - Args: - text: Decoded model output text (from ``tokenizer.decode(...)``). - - Returns: - A dict with keys: - - ``"thinking"``: The chain-of-thought text, or ``None`` if no - thinking delimiters were found. - - ``"answer"``: The final answer text. - - Example:: - - >>> from vllm.reasoning.gemma4_utils import parse_thinking_output - >>> output_text = tokenizer.decode(outputs[0], skip_special_tokens=False) - >>> result = parse_thinking_output(output_text) - >>> print(result["thinking"]) # chain-of-thought reasoning or None - >>> print(result["answer"]) # final answer - - """ - if _THINKING_END_TAG in text: - parts = text.split(_THINKING_END_TAG, 1) - thinking_block = parts[0] - answer = _clean_answer(parts[1]) - - # Extract thinking content: strip the start tag if present - if _THINKING_START_TAG in thinking_block: - thinking = thinking_block.split(_THINKING_START_TAG, 1)[1] - else: - thinking = thinking_block - - # Strip the "thought\n" channel role label the model emits inside - # <|channel>thought\n... (analogous to "user\n" in - # <|turn>user\n...). - thinking = _strip_thought_label(thinking.strip()) - thinking = thinking.strip() - - return {"thinking": thinking, "answer": answer} - - # No thinking delimiters found. - # Strip spurious "thought\n" role label that some Gemma4 models sometimes - # emit even without thinking mode enabled, then clean trailing tokens. - answer = _strip_thought_label(text) - answer = _clean_answer(answer) - return {"thinking": None, "answer": answer} - - -def _strip_thought_label(text: str) -> str: - """Strip the spurious ``thought\\n`` label from the start of text. - - Only strips when ``thought`` appears as the very first word followed by - a newline — preserving the word ``thought`` in any other context. - """ - if text.startswith("thought\n"): - return text[len("thought\n") :] - return text - - -def _clean_answer(text: str) -> str: - """Clean trailing sentinel tokens from the answer text. - - Strips ````, ````, and surrounding whitespace that the - model appends at the end of its response. - """ - text = text.strip() - # Strip trailing (Gemma4 turn-end marker) - if text.endswith(_TURN_END_TAG): - text = text[: -len(_TURN_END_TAG)].rstrip() - # Strip trailing if present - if text.endswith(""): - text = text[:-5].rstrip() - return text diff --git a/vllm/reasoning/glm47_moe_reasoning_parser.py b/vllm/reasoning/glm47_moe_reasoning_parser.py deleted file mode 100644 index 8e963f88b0..0000000000 --- a/vllm/reasoning/glm47_moe_reasoning_parser.py +++ /dev/null @@ -1,6 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from vllm.parser.engine.registered_adapters import Glm47MoeParserReasoningAdapter - -__all__ = ["Glm47MoeParserReasoningAdapter"] diff --git a/vllm/reasoning/granite_reasoning_parser.py b/vllm/reasoning/granite_reasoning_parser.py deleted file mode 100644 index dc6118224f..0000000000 --- a/vllm/reasoning/granite_reasoning_parser.py +++ /dev/null @@ -1,372 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from collections.abc import Sequence -from typing import TYPE_CHECKING - -import regex as re -from transformers import PreTrainedTokenizerBase - -from vllm.entrypoints.generate.base.protocol import DeltaMessage -from vllm.reasoning import ReasoningParser - -if TYPE_CHECKING: - from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest - from vllm.entrypoints.openai.responses.protocol import ResponsesRequest - - -class GraniteReasoningParser(ReasoningParser): - """Reasoning parser for IBM Granite. - - IBM granite models currently use "Here is my thought process:" - and "Here is my response:" to separate its thinking / response outputs. - """ - - def __init__(self, tokenizer: PreTrainedTokenizerBase, *args, **kwargs): - super().__init__(tokenizer, *args, **kwargs) - - # NOTE: There have been some observed occurrences of quantized - # instances of the current models using "Here's" instead of "Here is", - # so to be safe, we match on both. - self.think_start_expr = r"(?:Here's|Here is) my thought process:" - self.response_start_expr = r"(?:Here's|Here is) my response:" - - self.reasoning_regex = re.compile( - rf"{self.think_start_expr}(.*?){self.response_start_expr}(.*)", re.DOTALL - ) - - self.valid_think_starts = [ - "Here's my thought process:", - "Here is my thought process:", - ] - self.valid_response_starts = ["Here's my response:", "Here is my response:"] - - # Substrings to match for sequence boundaries on raw text - self.seq_boundary_end = ":" - self.seq_boundary_start = "Here" - - # The longest any thinking / start of response message can be - self.longest_think_start = max( - len(think_start) for think_start in self.valid_think_starts - ) - - def extract_reasoning( - self, model_output: str, request: "ChatCompletionRequest | ResponsesRequest" - ) -> tuple[str | None, str | None]: - """Extract the reasoning content & content sections, respectively. - If the sequence doesn't match what we expect, i.e., the model generates - something else, all content is considered non-reasoning content. - - Args: - model_output (str): Output of the model to be parsed. - request (ChatCompletionRequest): Request being processed. - - Returns: - tuple[Optional[str], Optional[str]]: Tuple pair containing the - reasoning content and non-reasoning content. - - """ - re_match = self.reasoning_regex.findall(model_output) - if not re_match: - return None, model_output - reasoning, response_content = re_match[0] - if not response_content: - return reasoning, None - return reasoning, response_content - - def extract_reasoning_streaming( - self, - previous_text: str, - current_text: str, - delta_text: str, - previous_token_ids: Sequence[int], - current_token_ids: Sequence[int], - delta_token_ids: Sequence[int], - ) -> DeltaMessage | None: - """Extract the reasoning content / content emitted by granite models; - If the sequence doesn't match what we expect, i.e., the model generates - something else, all content is considered non-reasoning content. - - NOTE: Granite models do not use a special token to start their reasoning - and response sections; instead they have token sequences, e.g., - - Here is my thought process: Foo Here is my response: Bar - - This increases the complexity of correctly handling streams, since we - need to watch for specific sequences and correctly parse them without - dropping content that is potentially overlapping & spanning multiple - delta messages. - - Args: - previous_text (str): Previous text outside of this delta message. - current_text (str): Previous text + delta text. - delta_text (str): Text to consider and parse content from. - previous_token_ids (Sequence[int]): Token IDs of previous_text. - current_token_ids (Sequence[int]): Token IDs of current_text. - delta_token_ids (Sequence[int]): Token IDs of delta_text. - - Returns: - Union[DeltaMessage, None] - DeltaMessage with either reasoning content or content, or None. - - """ - reasoning, resp_seq_len, content = self._get_content_sections(current_text) - # Either we haven't finished the start of the reasoning sequence, - # or the model is generating something unexpected. - if not reasoning: - delta_message = self._get_delta_message_with_no_reasoning_bounds( - current_text, delta_text - ) - # We have a start of reasoning message, but have not yet finished - # the start of response sequence. - elif not content: - delta_message = self._get_delta_message_with_no_response_bounds( - current_text, reasoning, delta_text - ) - # We've finished both the start of reasoning and start of response seq. - else: - # This should never happen since we matched on the response - assert resp_seq_len is not None - delta_message = self._get_delta_message_with_both_bounds( - delta_text, reasoning, content, current_text, resp_seq_len - ) - if not delta_message.content and not delta_message.reasoning: - return None - return delta_message - - #### Implementation details of stream parsing for granite models - def _is_reasoning_start_substr(self, text: str) -> bool: - """Check if a text matches one of the possible start reasoning seqs. - - Args: - text (str): Text to check for leading substr. - - Returns: - bool: True if any of the possible reasoning start seqs match. - - """ - return any( - think_start.startswith(text) for think_start in self.valid_think_starts - ) - - def _is_response_start_substr(self, text: str) -> bool: - """Check if a text matches one of the possible start response seqs. - - Args: - text (str): Text to check for leading substr. - - Returns: - bool: True if any of the possible response start seqs match. - - """ - return any( - response_start.startswith(text) - for response_start in self.valid_response_starts - ) - - def _get_delta_message_with_no_reasoning_bounds( - self, - current_text: str, - delta_text: str, - ) -> DeltaMessage: - """Parse the delta message when the current text has not yet completed - its start of reasoning sequence. - - Args: - current_text (str): The full previous + delta text. - delta_text (str): Text to consider and parse content from. - - Returns: - DeltaMessage: Message containing the parsed content. - - """ - prev_longest_length = len(current_text) - len(delta_text) - is_substr = self._is_reasoning_start_substr(current_text) - was_substr = self._is_reasoning_start_substr(current_text[:prev_longest_length]) - - # Check if we just generated something NOT in the special token seq; - # if so, add everything that we previously skipped with this delta - # message and append everything to content in the future. - if was_substr and not is_substr: - return DeltaMessage( - reasoning=None, - content=current_text, - ) - if is_substr: - # Might still be in the special token sequence; return nothing - return DeltaMessage(reasoning=None, content=None) - # Otherwise the sequence has already been broken and we already - # corrected; just return the delta text as normal content. - return DeltaMessage(reasoning=None, content=delta_text) - - def _get_delta_message_with_no_response_bounds( - self, - current_text: str, - reasoning: str, - delta_text: str, - ) -> DeltaMessage: - """Parse the delta message when the current text has both reasoning - content with no (response) content. NOTE that we may have overlapping - tokens with the start of reasoning / start of response sequences on - either side of the delta text. - - Args: - current_text (str): The full previous + delta text. - reasoning (str): reasoning content from current_text. - delta_text (str): Text to consider and parse content from. - - Returns: - DeltaMessage: Message containing the parsed content. - - """ - # If we have no reasoning content or explicitly end with the start of - # response sequence, we are in transition to the response; need to be - # careful here, since the final token (:) will match the reasoning - # content and fully parse it out; we should not pass the : back. - ends_with_start_response_seq = any( - current_text.endswith(response_start) - for response_start in self.valid_response_starts - ) - if reasoning is None or ends_with_start_response_seq: - return DeltaMessage(reasoning=None, content=None) - - # Consider previous / current text only within context of the reasoning - previous_text = reasoning[: -len(delta_text)] - current_text = reasoning - - # We need to be careful about adding unfinished response sequences; - # Find the place at which we MIGHT be starting a response sequence - prev_idx = previous_text.rfind(self.seq_boundary_start) - delta_idx = delta_text.rfind(self.seq_boundary_start) - - # Check the state of potential start of response substring matches. - prev_was_substr = ( - self._is_response_start_substr(previous_text[prev_idx:]) - if prev_idx >= 0 - else False - ) - delta_continues_substr = ( - self._is_response_start_substr(current_text[prev_idx:]) - if prev_idx >= 0 - else False - ) - delta_new_substr = ( - self._is_response_start_substr(delta_text[delta_idx:]) - if delta_idx >= 0 - else False - ) - - # Delta only contains potential continued response sequence text. - if delta_continues_substr: - return DeltaMessage(reasoning=None, content=None) - - if not prev_was_substr: - # Delta may be starting a new response seq but has other text too. - if delta_new_substr: - return DeltaMessage(reasoning=delta_text[:delta_idx], content=None) - # Normal case for most reasoning text (no potential special seqs). - return DeltaMessage(reasoning=delta_text, content=None) - # The substring that previously seemed to be a potential response - # seq wasn't one; we need to add the content to the delta message, - # and also slice off the potential response sequence - elif delta_new_substr: - reasoning = previous_text[prev_idx:] + delta_text[:delta_idx] - return DeltaMessage(reasoning=reasoning, content=None) - # No new substring yet, and we broke our old one; take the whole delta - return DeltaMessage( - reasoning=previous_text[prev_idx:] + delta_text, - content=None, - ) - - def _get_delta_message_with_both_bounds( - self, - delta_text: str, - reasoning: str, - response_content: str, - current_text: str, - response_seq_len: int, - ) -> DeltaMessage: - """Parse the delta message when the current text has both reasoning - content and normal (response) content. - - Args: - delta_text: Text to consider and parse content from. - reasoning: reasoning content from current_text. - response_content: response content from current_text. - current_text: The full previous + delta text. - response_seq_len: Len of the complete response sequence used. - - Returns: - DeltaMessage: Message containing the parsed content. - - """ - # Always have content; take length to the end - delta_content = delta_text[-len(response_content) :] - reasoning_end_idx = len(delta_text) - (len(response_content) + response_seq_len) - - if reasoning_end_idx < 0: - delta_reasoning = None - else: - # Get the starting offset - start_reasoning_idx = ( - len(reasoning) + response_seq_len + len(response_content) - 1 - ) - delta_offset = len(current_text) - len(delta_text) - start_offset = start_reasoning_idx - delta_offset - if start_offset < 0: - start_offset = 0 - delta_reasoning = delta_text[start_offset:reasoning_end_idx] - - return DeltaMessage( - reasoning=delta_reasoning, - content=delta_content, - ) - - def _get_content_sections( - self, current_text: str - ) -> tuple[str | None, int | None, str | None]: - """Parse the text to extract the reasoning content / content - if we have them. - - Args: - current_text (str): The full previous + delta text. - - Returns: - tuple[Optional[str], Optional[int], Optional[str]]: Tuple of len 3 - containing the reasoning content, the length of the response seq - (if there is one) and the non-reasoning content. - - """ - current_chunk_start = 0 - start_reasoning = None - parsed_content = False - delimiter_idxs = [ - idx - for idx, char in enumerate(current_text) - if char == self.seq_boundary_end - ] - - for current_chunk_end in delimiter_idxs: - current_chunk = current_text[current_chunk_start:current_chunk_end] - # Check to see if the start of reasoning seq if complete - if start_reasoning is None: - for think_start in self.valid_think_starts: - if current_chunk == think_start[:-1]: - start_reasoning = current_chunk_end + 1 - current_chunk_start = current_chunk_end + 1 - break - - # Check to see if the start of response seq if complete - elif not parsed_content: - for response_start in self.valid_response_starts: - if current_chunk[-len(response_start) + 1 :] == response_start[:-1]: - # Mark end of reasoning and start response content - # after the start of response sequence. - end_reasoning = current_chunk_end - len(response_start) - reasoning = current_text[start_reasoning:end_reasoning] - response_content = current_text[current_chunk_end + 1 :] - return reasoning, len(response_start), response_content - - if start_reasoning and not parsed_content: - return current_text[start_reasoning:], None, None - return None, None, None diff --git a/vllm/reasoning/granite_thinking_engine_reasoning_parser.py b/vllm/reasoning/granite_thinking_engine_reasoning_parser.py deleted file mode 100644 index 37c7d4b350..0000000000 --- a/vllm/reasoning/granite_thinking_engine_reasoning_parser.py +++ /dev/null @@ -1,8 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from vllm.parser.engine.registered_adapters import ( - GraniteThinkingParserReasoningAdapter, -) - -__all__ = ["GraniteThinkingParserReasoningAdapter"] diff --git a/vllm/reasoning/hunyuan_a13b_reasoning_parser.py b/vllm/reasoning/hunyuan_a13b_reasoning_parser.py deleted file mode 100644 index 0a7e021a46..0000000000 --- a/vllm/reasoning/hunyuan_a13b_reasoning_parser.py +++ /dev/null @@ -1,226 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from collections.abc import Sequence -from typing import TYPE_CHECKING - -import regex as re -from transformers import PreTrainedTokenizerBase - -from vllm.entrypoints.generate.base.protocol import DeltaMessage -from vllm.reasoning import ReasoningParser - -if TYPE_CHECKING: - from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest - from vllm.entrypoints.openai.responses.protocol import ResponsesRequest - - -class HunyuanA13BReasoningParser(ReasoningParser): - """Reasoning parser for Hunyuan A13B Model. - - HunyuanReasoningParser - - This class implements a reasoning parser specifically designed - for the Hunyuan A13B Model. It is responsible for parsing and - extracting structured reasoning and answer segments from model - outputs that follow a specific pattern. - - Key Features: - - For non-stream output , Recognizes and extracts reasoning ("think") - and answer ("answer") sections from text using regular expressions. - - For stream process, it requires a token id sequences to change the - reasoning state and other state so it maintains internal state to - manage parsing across multiple token. - - - think start: "\n": [14023, 771, 397] - think ends: "\n\n\n": [198, 524, 27963, 397, 27, 9399, 397] - response ends: "\n": [524, 9399, 29] - """ - - def __init__(self, tokenizer: PreTrainedTokenizerBase, *args, **kwargs): - super().__init__(tokenizer, *args, **kwargs) - self.think_start_expr = r"\n" - self.think_end_expr = r"\n\n" - - self.response_start_expr = r"\n\n\n" - self.response_end_expr = r"\n" - - self.full_match_reasoning_regex = re.compile( - rf"(?:{self.think_start_expr}(.*?){self.response_start_expr})?(.*?){self.response_end_expr}", - re.DOTALL, - ) - - self.half_match_reasoning_regex = re.compile( - rf"{self.think_start_expr}(.*?){self.response_start_expr}(.*)", re.DOTALL - ) - - self.think_start_ids = [14023, 771, 397] - self.think_start_ids_fast = [14023, 771, 1363] - self.response_start_ids = [198, 524, 27963, 397, 27, 9399, 397] - self.response_start_ids_fast = [524, 27963, 397, 27, 9399, 397] - self.response_end_ids = [198, 524, 9399, 29] - self.fast_think_ids = [14023, 771, 1363, 524, 27963, 397, 27, 9399, 397] - - # when state change, send out all the buffered text in last state - self.buffered_text: list[str] = [] - self.buffered_ids: list[int] = [] - - self.current_state = "reasoning" - self.all_states = ["reasoning", "response"] - - self.current_state = "idle" - self.expected_sequence = self.think_start_ids - # this sequence only for the think start, it has two way to start. - self.expected_sequence_side = self.think_start_ids_fast - self.sequence_index = 0 - self.token_buffer: list[int] = [] - self.text_buffer = "" - - def is_reasoning_end(self, input_ids: Sequence[int]) -> bool: - return self.current_state == "response" - - def extract_content_ids(self, input_ids: list[int]) -> list[int]: - # for hunyuan streaming reason parsing, the stream parse - # will call first, and the same token will be called in - # is_reasoning_end and extract_content_ids - # this id is not part of content, so just return [] here. - return [] - - def extract_reasoning( - self, model_output: str, request: "ChatCompletionRequest | ResponsesRequest" - ) -> tuple[str | None, str | None]: - """Extract the reasoning content & content sections, respectively. - If the sequence doesn't match what we expect, i.e., the model generates - something else, all content is considered non-reasoning content. - - Args: - model_output (str): Output of the model to be parsed. - request (ChatCompletionRequest): Request being processed. - - Returns: - tuple[Optional[str], Optional[str]]: Tuple pair containing the - reasoning content and non-reasoning content. - - """ - re_match = self.full_match_reasoning_regex.findall(model_output) - if re_match: - reasoning, response_content = re_match[0] - if len(reasoning) == 0: - reasoning = None - if len(response_content) == 0: - response_content = None - return reasoning, response_content - - fallback_regex = self.half_match_reasoning_regex - fallback_match = fallback_regex.findall(model_output) - if fallback_match: - reasoning, response_content = fallback_match[0] - - if response_content.endswith(self.response_end_expr): - response_content = response_content[: -len(self.response_end_expr)] - - if len(reasoning) == 0: - reasoning = None - if len(response_content) == 0: - response_content = None - - return reasoning, response_content - - return None, model_output - - def extract_reasoning_streaming( - self, - previous_text: str, - current_text: str, - delta_text: str, - previous_token_ids: Sequence[int], - current_token_ids: Sequence[int], - delta_token_ids: Sequence[int], - ) -> DeltaMessage | None: - """Extract content using token ID sequence state machine.""" - # Define sequences - think_start_sequence = self.think_start_ids - response_start_sequence = self.response_start_ids - response_end_sequence = self.response_end_ids - - assert len(delta_token_ids) == 1 - # Process each token in the delta - token = delta_token_ids[0] - - def check_token_with_sequence(token): - if self.current_state == "idle" or self.current_state == "think": - return ( - token == self.expected_sequence[self.sequence_index] - or token == self.expected_sequence_side[self.sequence_index] - ) - else: - return token == self.expected_sequence[self.sequence_index] - - def check_last_token(token): - if self.current_state == "idle" or self.current_state == "think": - # only return true if it's judge using a side sequence. - if ( - self.sequence_index - 1 < len(self.expected_sequence_side) - and token == self.expected_sequence_side[self.sequence_index - 1] - ): - return self.sequence_index == len(self.expected_sequence_side) - else: - return self.sequence_index == len(self.expected_sequence) - else: - return self.sequence_index == len(self.expected_sequence) - - # Check if token matches expected sequence - token_in_state_seq = check_token_with_sequence(token) - - if token_in_state_seq: - # Store matching token - self.token_buffer.append(token) - self.text_buffer += delta_text - self.sequence_index += 1 - ## state change from idle->think->response->idle - - # Check if sequence fully matched - if check_last_token(token): - # State transition - if self.current_state == "idle": - self.current_state = "think" - self.expected_sequence = response_start_sequence - self.expected_sequence_side = self.response_start_ids_fast - elif self.current_state == "think": - self.current_state = "response" - self.expected_sequence = response_end_sequence - elif self.current_state == "response": - self.current_state = "idle" - self.expected_sequence = think_start_sequence - self.expected_sequence_side = self.think_start_ids_fast - - # Reset matching state - self.sequence_index = 0 - self.token_buffer = [] - self.text_buffer = "" - # Do not send content for state transition texts. - else: - # Sequence broken - handle buffered content - if self.token_buffer and len(self.token_buffer) > 0: - # Send buffered tokens - buffered_content = self.text_buffer + delta_text - # Reset matching state - self.sequence_index = 0 - self.token_buffer = [] - self.text_buffer = "" - - # Return content based on current state - if self.current_state == "think": - return DeltaMessage(reasoning=buffered_content, content=None) - else: - return DeltaMessage(reasoning=None, content=buffered_content) - else: - # No buffered content, send normally - if self.current_state == "think": - return DeltaMessage(reasoning=delta_text, content=None) - else: - return DeltaMessage(reasoning=None, content=delta_text) - - # If no content to send in this delta - return None diff --git a/vllm/reasoning/hy_v3_reasoning_parser.py b/vllm/reasoning/hy_v3_reasoning_parser.py deleted file mode 100644 index 77a1a271d4..0000000000 --- a/vllm/reasoning/hy_v3_reasoning_parser.py +++ /dev/null @@ -1,143 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from collections.abc import Iterable, Sequence -from typing import TYPE_CHECKING - -from vllm.entrypoints.generate.base.protocol import DeltaMessage -from vllm.logger import init_logger -from vllm.reasoning.basic_parsers import BaseThinkingReasoningParser -from vllm.reasoning.identity_reasoning_parser import IdentityReasoningParser -from vllm.tokenizers import TokenizerLike - -if TYPE_CHECKING: - from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest - from vllm.entrypoints.openai.responses.protocol import ResponsesRequest - -logger = init_logger(__name__) - - -class HYV3ReasoningParser(BaseThinkingReasoningParser): - """HYV3 parser that delegates to either HYV3ReasoningParser or - IdentityReasoningParser based on `reasoning_effort`. - - The HYV3 model uses ... tokens to denote reasoning text. - This parser extracts the reasoning content from the model output. - """ - - def __init__(self, tokenizer: TokenizerLike, *args, **kwargs): - init_kwargs = getattr(tokenizer, "init_kwargs", None) or {} - self.suffix: str = init_kwargs.get("token_suffix") or "" - super().__init__(tokenizer, *args, **kwargs) - - # First, If there is reasoning_effort in chat_kwargs, - # prioritize using chat_kwargs.reasoning_effort. - # If it's not present, use the "reasoning_effort" field - # at the outer level of the chat message. - # Otherwise, If both are empty, assign "no_think". - - chat_kwargs = kwargs.get("chat_template_kwargs", {}) or {} - reasoning_effort = ( - chat_kwargs.get("reasoning_effort") - or kwargs.get("reasoning_effort") - or "no_think" - ) - - logger.debug("reasoning_effort for choosing parser: %s", reasoning_effort) - - self._identity_parser: IdentityReasoningParser | None - if reasoning_effort == "no_think": - self._identity_parser = IdentityReasoningParser(tokenizer, *args, **kwargs) - else: - self._identity_parser = None - - @property - def start_token(self) -> str: - """The token that starts reasoning content.""" - return f"" - - @property - def end_token(self) -> str: - """The token that ends reasoning content.""" - return f"" - - def is_reasoning_end(self, input_ids: Sequence[int]) -> bool: - if self._identity_parser is not None: - return self._identity_parser.is_reasoning_end(input_ids) - - return super().is_reasoning_end(input_ids) - - def is_reasoning_end_streaming( - self, input_ids: Sequence[int], delta_ids: Iterable[int] - ) -> bool: - if self._identity_parser is not None: - return self._identity_parser.is_reasoning_end_streaming( - input_ids, delta_ids - ) - - return super().is_reasoning_end_streaming(input_ids, delta_ids) - - def extract_content_ids(self, input_ids: list[int]) -> list[int]: - if self._identity_parser is not None: - return self._identity_parser.extract_content_ids(input_ids) - - return super().extract_content_ids(input_ids) - - def extract_reasoning( - self, model_output: str, request: "ChatCompletionRequest | ResponsesRequest" - ) -> tuple[str | None, str | None]: - if self._identity_parser is not None: - return self._identity_parser.extract_reasoning(model_output, request) - - return super().extract_reasoning(model_output, request) - - def extract_reasoning_streaming( - self, - previous_text: str, - current_text: str, - delta_text: str, - previous_token_ids: Sequence[int], - current_token_ids: Sequence[int], - delta_token_ids: Sequence[int], - ) -> DeltaMessage | None: - if self._identity_parser is not None: - return self._identity_parser.extract_reasoning_streaming( - previous_text, - current_text, - delta_text, - previous_token_ids, - current_token_ids, - delta_token_ids, - ) - - ret = super().extract_reasoning_streaming( - previous_text, - current_text, - delta_text, - previous_token_ids, - current_token_ids, - delta_token_ids, - ) - if ( - ret is not None - and self.start_token_id not in previous_token_ids - and self.start_token_id not in delta_token_ids - ): - if self.end_token_id in delta_token_ids: - # end token in delta with more tokens, - # extract reasoning content and content - end_index = delta_text.find(self.end_token) - reasoning = delta_text[:end_index] - content = delta_text[end_index + len(self.end_token) :] - return DeltaMessage( - reasoning=reasoning, - content=content if content else None, - ) - elif self.end_token_id in previous_token_ids: - # end token in previous, thinking content ends - return DeltaMessage(content=delta_text) - else: - # no end token in previous or delta, reasoning content continues - return DeltaMessage(reasoning=delta_text) - - return ret diff --git a/vllm/reasoning/hy_v4_reasoning_parser.py b/vllm/reasoning/hy_v4_reasoning_parser.py deleted file mode 100644 index eabe6c31ae..0000000000 --- a/vllm/reasoning/hy_v4_reasoning_parser.py +++ /dev/null @@ -1,323 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from __future__ import annotations - -from collections.abc import Iterable, Sequence -from typing import TYPE_CHECKING, TypedDict - -import regex as re - -from vllm.entrypoints.generate.base.protocol import DeltaMessage -from vllm.logger import init_logger -from vllm.reasoning import ReasoningParser -from vllm.tokenizers import TokenizerLike - -if TYPE_CHECKING: - from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest - from vllm.entrypoints.openai.responses.protocol import ResponsesRequest - -logger = init_logger(__name__) - -# Must match the chat template's own default for ``reasoning_effort``. -DEFAULT_REASONING_EFFORT = "high" - - -class ReasoningDelta(TypedDict): - """One streaming reasoning delta. Either/both fields may be None.""" - - reasoning: str | None - content: str | None - - -# NOTE: mirrored in ``vllm.tool_parsers.hy_v4_tool_parser`` (same pattern as -# ``gemma4_utils``) so neither package depends on the other. -def detect_token_suffix(tokenizer: TokenizerLike) -> str: - """Detect the per-checkpoint suffix used by Hunyuan structural tokens. - - Args: - tokenizer: Tokenizer of the served checkpoint. - - Returns: - The suffix including its leading colon (e.g. ``":6124c78e"``), or ``""`` - when the checkpoint uses unsuffixed tokens. - - Raises: - RuntimeError: The tokenizer declares the structural tokens through - ``model_specific_special_tokens``, which transformers 5 no longer - round-trips. - - """ - init_kwargs = getattr(tokenizer, "init_kwargs", None) or {} - think_begin_as_special = init_kwargs.get("model_specific_special_tokens", {}).get( - "think_begin_token", "" - ) - if think_begin_as_special: - raise RuntimeError( - "This checkpoint declares HYV4 structural tokens (think_begin_token" - "/toolcalls_begin_token/argkey_begin_token) in " - "tokenizer_config.json, which transformers 5 no longer supports. " - "Remove those fields and keep the tokens in the tokenizer's own " - "token definitions so the suffix can be read from the vocab." - ) - - structural_token_re = re.compile( - r"<(?:think|tool_calls|tool_call|arg_key|arg_value)(:[^\s>]+)?>" - ) - for token in tokenizer.get_vocab(): - match = structural_token_re.fullmatch(token) - if match: - return match.group(1) or "" - - return "" - - -class HYV4ReasoningExtractor: - """Reasoning extraction for HYV4, on plain data. - - ``thinking`` selects the mode: - - ``True``: parse ``...``; the start token is injected at the - end of the prompt and is therefore usually absent from the output. - - ``False``: ``no_think`` -- the whole output is content, no reasoning. - """ - - def __init__(self, vocab: dict[str, int], token_suffix: str, thinking: bool = True): - self.thinking = thinking - self.start_token: str = f"" - self.end_token: str = f"" - - start_token_id = vocab.get(self.start_token) - end_token_id = vocab.get(self.end_token) - if thinking and (start_token_id is None or end_token_id is None): - raise RuntimeError( - "HYV4 reasoning extractor could not locate think " - "start/end tokens in the tokenizer!" - ) - # In no_think mode the ids are unused (kept for symmetry / debugging). - self.start_token_id: int | None = start_token_id - self.end_token_id: int | None = end_token_id - - def has_reasoning_ended(self, token_ids: Sequence[int]) -> bool: - """Idempotent STATE query: has ```` been emitted yet? - - Monotonic -- once the end token is present this stays True. Use for - "has reasoning finished, given everything so far". (no_think is always - ended.) Contrast with ``reasoning_ended_in_delta`` (edge detector). - """ - if not self.thinking: - return True - for i in range(len(token_ids) - 1, -1, -1): - if token_ids[i] == self.start_token_id: - return False - if token_ids[i] == self.end_token_id: - return True - return False - - def reasoning_ended_in_delta( - self, token_ids: Sequence[int], delta_ids: Iterable[int] - ) -> bool: - """Edge detector (NOT idempotent): did reasoning end *in this* step? - - True only on the step whose ``delta_ids`` contains ````. Callers - use it once behind their own "already ended" latch, so it need not be - monotonic; it only checks the new delta (cheaper than scanning the whole - sequence). Contrast with ``has_reasoning_ended`` (cumulative state). - """ - if not self.thinking: - return True - end_token_id = self.end_token_id - if end_token_id is None: - return False - return end_token_id in delta_ids - - def extract_content_ids(self, input_ids: list[int]) -> list[int]: - """Return the token ids after ```` (all of them in no_think).""" - if not self.thinking: - return input_ids - head = input_ids[: max(0, len(input_ids) - 1)] - if self.end_token_id not in head: - return [] - return input_ids[input_ids.index(self.end_token_id) + 1 :] - - def count_reasoning_tokens(self, token_ids: Sequence[int]) -> int: - """Count reasoning tokens (0 in no_think). - - The ```` start token is injected at the END of the prompt, so it - is normally absent from ``token_ids``. Only skip a leading start token - when it actually appears first (legacy / no prompt-injection); otherwise - count every token up to ````. - """ - if not self.thinking: - return 0 - start = 1 if (token_ids and token_ids[0] == self.start_token_id) else 0 - try: - return token_ids.index(self.end_token_id, start) - start - except ValueError: - # No end token -- reasoning was truncated; everything counts. - return len(token_ids) - start - - def extract_reasoning(self, model_output: str) -> tuple[str | None, str | None]: - """Split a complete output into ``(reasoning, content)``. - - In no_think mode reasoning is None and the whole output is content. The - start token may be absent (injected in the prompt), so reasoning is - assumed to begin at the start of the output. - """ - if not self.thinking: - return None, model_output - - parts = model_output.partition(self.start_token) - model_output = parts[2] if parts[1] else parts[0] - - if self.end_token not in model_output: - return model_output, None - reasoning, _, content = model_output.partition(self.end_token) - return reasoning, content or None - - def extract_reasoning_streaming( - self, - previous_text: str, - current_text: str, - delta_text: str, - previous_token_ids: Sequence[int], - current_token_ids: Sequence[int], - delta_token_ids: Sequence[int], - ) -> ReasoningDelta | None: - """Classify a streaming delta as reasoning and/or content. - - In no_think mode every delta is content. Otherwise: same logic as vLLM's - generic thinking parser, except the final branch (no start token seen - anywhere) treats output as reasoning-then-content, because HYV4 injects - ```` in the prompt (absent from the output). - """ - if not self.thinking: - if not delta_text: - return None - return ReasoningDelta(reasoning=None, content=delta_text) - - # Skip a delta that is a single lone start/end special token. - if len(delta_token_ids) == 1 and delta_token_ids[0] in ( - self.start_token_id, - self.end_token_id, - ): - return None - - if self.start_token_id in previous_token_ids: - if self.end_token_id in delta_token_ids: - end_index = delta_text.find(self.end_token) - reasoning = delta_text[:end_index] - content = delta_text[end_index + len(self.end_token) :] - return ReasoningDelta(reasoning=reasoning, content=content or None) - elif self.end_token_id in previous_token_ids: - return ReasoningDelta(reasoning=None, content=delta_text) - else: - return ReasoningDelta(reasoning=delta_text, content=None) - elif self.start_token_id in delta_token_ids: - if self.end_token_id in delta_token_ids: - start_index = delta_text.find(self.start_token) - end_index = delta_text.find(self.end_token) - reasoning = delta_text[start_index + len(self.start_token) : end_index] - content = delta_text[end_index + len(self.end_token) :] - return ReasoningDelta(reasoning=reasoning, content=content or None) - else: - return ReasoningDelta(reasoning=delta_text, content=None) - else: - # HYV4: was injected at the end of the prompt, so it never - # appears in the output. Treat the stream as reasoning until - # , then content. - if self.end_token_id in delta_token_ids: - end_index = delta_text.find(self.end_token) - reasoning = delta_text[:end_index] - content = delta_text[end_index + len(self.end_token) :] - return ReasoningDelta(reasoning=reasoning, content=content or None) - elif self.end_token_id in previous_token_ids: - return ReasoningDelta(reasoning=None, content=delta_text) - else: - return ReasoningDelta(reasoning=delta_text, content=None) - - -def build_reasoning_extractor( - tokenizer: TokenizerLike, *, thinking: bool = True -) -> HYV4ReasoningExtractor: - return HYV4ReasoningExtractor( - tokenizer.get_vocab(), detect_token_suffix(tokenizer), thinking - ) - - -class HYV4ReasoningParser(ReasoningParser): - """vLLM adapter around :class:`HYV4ReasoningExtractor`. - - Decides ``thinking`` from the request's ``reasoning_effort`` and converts - the extractor's ``ReasoningDelta`` to ``DeltaMessage``. - """ - - def __init__(self, tokenizer: TokenizerLike, *args, **kwargs): - super().__init__(tokenizer, *args, **kwargs) - - # Decide thinking vs no_think from the request; the extractor handles - # both modes internally. Prefer chat_template_kwargs.reasoning_effort, - # then the top-level reasoning_effort. The default must match the chat - # template, which sets reasoning_effort='high' when the request omits - # it -- defaulting to no_think here would return the whole - # chain-of-thought (and a stray ````) as content. - chat_kwargs = kwargs.get("chat_template_kwargs", {}) or {} - reasoning_effort = chat_kwargs.get("reasoning_effort", None) - if not reasoning_effort: - reasoning_effort = ( - kwargs.get("reasoning_effort") or DEFAULT_REASONING_EFFORT - ) - logger.debug("reasoning_effort for choosing parser: %s", reasoning_effort) - thinking = reasoning_effort != "no_think" - - self._extractor = build_reasoning_extractor(tokenizer, thinking=thinking) - - # Reported regardless of the per-request thinking mode: ``ReasoningConfig`` - # is engine-wide and needs the checkpoint's delimiters to derive the token - # ids used for thinking budget enforcement. - @property - def reasoning_start_str(self) -> str: - return self._extractor.start_token - - @property - def reasoning_end_str(self) -> str: - return self._extractor.end_token - - def is_reasoning_end(self, input_ids) -> bool: - return self._extractor.has_reasoning_ended(input_ids) - - def is_reasoning_end_streaming(self, input_ids, delta_ids) -> bool: - return self._extractor.reasoning_ended_in_delta(input_ids, delta_ids) - - def extract_content_ids(self, input_ids) -> list[int]: - return self._extractor.extract_content_ids(input_ids) - - def count_reasoning_tokens(self, token_ids) -> int: - return self._extractor.count_reasoning_tokens(token_ids) - - def extract_reasoning( - self, - model_output: str, - request: ChatCompletionRequest | ResponsesRequest, - ) -> tuple[str | None, str | None]: - return self._extractor.extract_reasoning(model_output) - - def extract_reasoning_streaming( - self, - previous_text: str, - current_text: str, - delta_text: str, - previous_token_ids, - current_token_ids, - delta_token_ids, - ) -> DeltaMessage | None: - delta = self._extractor.extract_reasoning_streaming( - previous_text, - current_text, - delta_text, - previous_token_ids, - current_token_ids, - delta_token_ids, - ) - if delta is None: - return None - return DeltaMessage(reasoning=delta["reasoning"], content=delta["content"]) diff --git a/vllm/reasoning/identity_reasoning_parser.py b/vllm/reasoning/identity_reasoning_parser.py deleted file mode 100644 index f2c9a1e21e..0000000000 --- a/vllm/reasoning/identity_reasoning_parser.py +++ /dev/null @@ -1,72 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from collections.abc import Iterable, Sequence -from typing import TYPE_CHECKING - -from transformers import PreTrainedTokenizerBase - -from vllm.entrypoints.generate.base.protocol import DeltaMessage -from vllm.reasoning import ReasoningParser - -if TYPE_CHECKING: - from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest - from vllm.entrypoints.openai.responses.protocol import ResponsesRequest - - -class IdentityReasoningParser(ReasoningParser): - """Identity reasoning parser. - - This parser does not attempt to parse or strip out reasoning tokens. - It treats the entire model output as content and ignores reasoning. - """ - - def __init__(self, tokenizer: PreTrainedTokenizerBase, *args, **kwargs): - super().__init__(tokenizer, *args, **kwargs) - if not self.model_tokenizer: - raise ValueError( - "The model tokenizer must be passed to the ReasoningParser " - "constructor during construction." - ) - - @property - def reasoning_start_str(self) -> str | None: - return None - - @property - def reasoning_end_str(self) -> str | None: - return None - - def is_reasoning_end(self, input_ids: Sequence[int]) -> bool: - # Always return True, since we never treat reasoning specially - return True - - def is_reasoning_end_streaming( - self, input_ids: Sequence[int], delta_ids: Iterable[int] - ) -> bool: - return True - - def extract_content_ids(self, input_ids: list[int]) -> list[int]: - # Identity: return all tokens as content - return input_ids - - def extract_reasoning_streaming( - self, - previous_text: str, - current_text: str, - delta_text: str, - previous_token_ids: Sequence[int], - current_token_ids: Sequence[int], - delta_token_ids: Sequence[int], - ) -> DeltaMessage | None: - # Just wrap delta_text as content, ignore reasoning - if delta_text: - return DeltaMessage(content=delta_text) - return None - - def extract_reasoning( - self, model_output: str, request: "ChatCompletionRequest | ResponsesRequest" - ) -> tuple[str | None, str | None]: - # No reasoning separation: return None for reasoning, - # and full model_output as content - return None, model_output diff --git a/vllm/reasoning/inkling_reasoning_parser.py b/vllm/reasoning/inkling_reasoning_parser.py deleted file mode 100644 index b9246e6c58..0000000000 --- a/vllm/reasoning/inkling_reasoning_parser.py +++ /dev/null @@ -1,6 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from vllm.parser.engine.registered_adapters import InklingParserReasoningAdapter - -__all__ = ["InklingParserReasoningAdapter"] diff --git a/vllm/reasoning/k2_horizon_reasoning_parser.py b/vllm/reasoning/k2_horizon_reasoning_parser.py deleted file mode 100644 index 4c24c6d1c4..0000000000 --- a/vllm/reasoning/k2_horizon_reasoning_parser.py +++ /dev/null @@ -1,159 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from collections.abc import Iterable, Sequence - -from vllm.entrypoints.generate.base.protocol import DeltaMessage -from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest -from vllm.entrypoints.openai.responses.protocol import ResponsesRequest -from vllm.reasoning.deepseek_r1_reasoning_parser import DeepSeekR1ReasoningParser -from vllm.tokenizers import TokenizerLike -from vllm.tool_parsers.utils import partial_tag_overlap - - -class K2HorizonReasoningParser(DeepSeekR1ReasoningParser): - """Reasoning parser for K2 Horizon.""" - - _TOOL_CALLS_START = "" - _EFFORT_TOKENS: dict[str, tuple[str, str]] = { - "high": ("", ""), - "medium": ("", ""), - "low": ("", ""), - } - - def __init__(self, tokenizer: TokenizerLike, *args, **kwargs) -> None: - chat_template_kwargs = kwargs.get("chat_template_kwargs", {}) or {} - effort = chat_template_kwargs.get("reasoning_effort", "high") - if not isinstance(effort, str) or effort not in self._EFFORT_TOKENS: - supported = ", ".join(self._EFFORT_TOKENS) - raise ValueError( - f"Unsupported reasoning_effort {effort!r}. " - f"Supported values: {supported}." - ) - - self._start_token, self._end_token = self._EFFORT_TOKENS[effort] - super().__init__(tokenizer, *args, **kwargs) - self._tool_calls_start_id = self.vocab.get(self._TOOL_CALLS_START) - self._stream_buffer = "" - self._stream_at_start = True - self._stream_finished = False - - @property - def start_token(self) -> str: - return self._start_token - - @property - def end_token(self) -> str: - return self._end_token - - def _without_generated_start(self, model_output: str) -> str: - return model_output.removeprefix(self.start_token) - - def _split_model_output(self, model_output: str) -> tuple[str, str | None]: - output = self._without_generated_start(model_output) - if self.end_token in output: - reasoning, _, content = output.partition(self.end_token) - return reasoning, content or None - - tool_start = output.find(self._TOOL_CALLS_START) - if tool_start != -1: - return output[:tool_start], output[tool_start:] - - return "", output or None - - def extract_reasoning( - self, - model_output: str, - request: ChatCompletionRequest | ResponsesRequest, - ) -> tuple[str, str | None]: - del request - return self._split_model_output(model_output) - - def is_reasoning_end(self, input_ids: Sequence[int]) -> bool: - for token_id in reversed(input_ids): - if token_id == self.start_token_id: - return False - if token_id == self.end_token_id or token_id == self._tool_calls_start_id: - return True - return False - - def is_reasoning_end_streaming( - self, - input_ids: Sequence[int], - delta_ids: Iterable[int], - ) -> bool: - del input_ids - return self._stream_finished or super().is_reasoning_end_streaming( - (), delta_ids - ) - - def extract_content_ids(self, input_ids: list[int]) -> list[int]: - if self.end_token_id in input_ids: - return input_ids[input_ids.index(self.end_token_id) + 1 :] - if ( - self._tool_calls_start_id is not None - and self._tool_calls_start_id in input_ids - ): - return input_ids[input_ids.index(self._tool_calls_start_id) :] - return input_ids if self._stream_finished else [] - - def _strip_stream_start(self) -> bool: - if not self._stream_at_start: - return True - if self.start_token.startswith(self._stream_buffer): - if self._stream_buffer != self.start_token: - return False - self._stream_buffer = "" - elif self._stream_buffer.startswith(self.start_token): - self._stream_buffer = self._stream_buffer[len(self.start_token) :] - self._stream_at_start = False - return True - - def extract_reasoning_streaming( - self, - previous_text: str, - current_text: str, - delta_text: str, - previous_token_ids: Sequence[int], - current_token_ids: Sequence[int], - delta_token_ids: Sequence[int], - ) -> DeltaMessage | None: - del ( - previous_text, - current_text, - previous_token_ids, - current_token_ids, - delta_token_ids, - ) - - if self._stream_finished: - return DeltaMessage(content=delta_text) if delta_text else None - - self._stream_buffer += delta_text - if not self._strip_stream_start(): - return None - - explicit_end = self._stream_buffer.find(self.end_token) - tool_start = self._stream_buffer.find(self._TOOL_CALLS_START) - if explicit_end != -1: - reasoning = self._stream_buffer[:explicit_end] - content = self._stream_buffer[explicit_end + len(self.end_token) :] - elif tool_start != -1: - reasoning = self._stream_buffer[:tool_start] - content = self._stream_buffer[tool_start:] - else: - overlap = max( - partial_tag_overlap(self._stream_buffer, self.end_token), - partial_tag_overlap(self._stream_buffer, self._TOOL_CALLS_START), - ) - sendable_end = len(self._stream_buffer) - overlap - reasoning = self._stream_buffer[:sendable_end] - self._stream_buffer = self._stream_buffer[sendable_end:] - return DeltaMessage(reasoning=reasoning) if reasoning else None - - self._stream_buffer = "" - self._stream_finished = True - return DeltaMessage( - reasoning=reasoning, - content=content or None, - ) diff --git a/vllm/reasoning/kimi_k2_reasoning_parser.py b/vllm/reasoning/kimi_k2_reasoning_parser.py deleted file mode 100644 index 45f99965bc..0000000000 --- a/vllm/reasoning/kimi_k2_reasoning_parser.py +++ /dev/null @@ -1,8 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from vllm.parser.engine.registered_adapters import KimiK2ParserReasoningAdapter - -KimiK2ReasoningParser = KimiK2ParserReasoningAdapter - -__all__ = ["KimiK2ReasoningParser"] diff --git a/vllm/reasoning/kimi_k3_reasoning_parser.py b/vllm/reasoning/kimi_k3_reasoning_parser.py deleted file mode 100644 index 1a1c62ba6d..0000000000 --- a/vllm/reasoning/kimi_k3_reasoning_parser.py +++ /dev/null @@ -1,468 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Reasoning parser for the Kimi K3 (XTML) chat format. - -This strips the ``think`` channel out of generated text and hands the remainder -(``response`` + ``tools`` channels) downstream. Kimi K3 wraps the thinking -channel as an XTML element built from special tokens:: - - <|open|>think<|sep|> <|close|>think<|sep|> - -Two subtleties drive the implementation: - * Unlike Kimi-K2 (a single ```` token), each K3 marker is a 3-token - sequence, so the token-id helpers search for the marker *subsequence* - rather than a single id. - * In thinking mode the serving layer may feed ``<|open|>think<|sep|>`` as the - generation prefix, so the model's output can begin *inside* the think - channel with no open marker. The text paths therefore treat a missing open - marker as "reasoning starts at offset 0". - -When thinking is disabled (``chat_template_kwargs={"thinking": False}`` or -``{"enable_thinking": False}``, i.e. instruct mode) the parser returns every -delta as normal content; there is simply no think channel to extract. -""" - -from collections.abc import Iterable, Sequence -from typing import TYPE_CHECKING - -import regex as re -from transformers import PreTrainedTokenizerBase - -from vllm.entrypoints.generate.base.protocol import DeltaMessage -from vllm.reasoning import ReasoningParser - -if TYPE_CHECKING: - from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest - from vllm.entrypoints.openai.responses.protocol import ResponsesRequest - - -def _match_at(haystack: Sequence[int], i: int, needle: Sequence[int]) -> bool: - """Whether *needle* occurs in *haystack* starting at *i*. - - Compares element by element rather than slicing: ``haystack`` is usually a - ``ConstantList``, whose slices cost a Python ``__getitem__`` plus a list - allocation on every probe. - """ - return all(haystack[i + k] == needle[k] for k in range(len(needle))) - - -def _subseq_index(haystack: Sequence[int], needle: Sequence[int]) -> int: - """Return start index of the last occurrence of needle in haystack, or -1.""" - n = len(needle) - if n == 0: - return -1 - first = needle[0] - for i in range(len(haystack) - n, -1, -1): - if haystack[i] == first and _match_at(haystack, i, needle): - return i - return -1 - - -def _newest_marker(haystack: Sequence[int], a: Sequence[int], b: Sequence[int]) -> int: - """Report which of *a* / *b* occurs last in *haystack*. - - Returns 0 if *a* is the newest marker, 1 if *b* is, -1 if neither occurs. - Equivalent to comparing two ``_subseq_index`` results, but a single backward - pass that stops at the first hit instead of walking to index 0 twice. - """ - if not a or not b: - return -1 - a0, b0 = a[0], b[0] - for i in range(len(haystack) - min(len(a), len(b)), -1, -1): - head = haystack[i] - if head == a0 and i + len(a) <= len(haystack) and _match_at(haystack, i, a): - return 0 - if head == b0 and i + len(b) <= len(haystack) and _match_at(haystack, i, b): - return 1 - return -1 - - -class KimiK3ReasoningParser(ReasoningParser): - """Reasoning parser for the Kimi K3 (XTML) think channel.""" - - def __init__(self, tokenizer: PreTrainedTokenizerBase, *args, **kwargs): - super().__init__(tokenizer) - - if not self.model_tokenizer: - raise ValueError( - "The model tokenizer must be passed to the ReasoningParser " - "constructor during construction." - ) - - # thinking can be disabled via chat_template_kwargs -> identity fallthrough - chat_kwargs = kwargs.get("chat_template_kwargs", {}) or {} - thinking = chat_kwargs.get("thinking", None) - if thinking is None: - thinking = chat_kwargs.get("enable_thinking", True) - self._thinking_enabled = bool(thinking) - - # XTML markers as literal strings (skip_special_tokens=False at serve time) - self._think_open = "<|open|>think<|sep|>" - self._think_close = "<|close|>think<|sep|>" - - # Content-channel markers that must be stripped from the final output. - # The tool parser handles these when active, but when tool_choice is - # "none" (the default for requests without tools) the tool parser is - # bypassed and the reasoning parser must strip them itself. - self._response_open = "<|open|>response<|sep|>" - self._response_close = "<|close|>response<|sep|>" - self._message_close = "<|close|>message<|sep|>" - - # Tolerant matchers: defense-in-depth against vLLM's added-token spacing - # ("<|open|> think <|sep|>"). The `\s*` is a no-op on clean input, so the - # normal path stays byte-exact; it only helps if some serving path leaves - # spaces_between_special_tokens on. See adjust_request for the real fix. - open_marker = r"<\|open\|>" - close_marker = r"<\|close\|>" - sep_marker = r"<\|sep\|>" - self._think_open_re = re.compile(open_marker + r"\s*think\s*" + sep_marker) - self._think_close_re = re.compile(close_marker + r"\s*think\s*" + sep_marker) - self._response_open_re = re.compile( - open_marker + r"\s*response\s*" + sep_marker - ) - self._response_close_re = re.compile( - close_marker + r"\s*response\s*" + sep_marker - ) - self._message_close_re = re.compile( - close_marker + r"\s*message\s*" + sep_marker - ) - - # marker token-id subsequences (each marker is 3 tokens) - self._think_open_ids = tokenizer.encode( - self._think_open, add_special_tokens=False - ) - self._think_close_ids = tokenizer.encode( - self._think_close, add_special_tokens=False - ) - self._response_open_ids = tokenizer.encode( - self._response_open, add_special_tokens=False - ) - self._last_streaming_delta_token_ids: tuple[int, ...] | None = None - self._last_streaming_content_token_ids: list[int] | None = None - - @property - def reasoning_start_str(self) -> str | None: - return self._think_open - - @property - def reasoning_end_str(self) -> str | None: - return self._think_close - - def adjust_request( - self, - request: "ChatCompletionRequest | ResponsesRequest", - ) -> "ChatCompletionRequest | ResponsesRequest": - request.skip_special_tokens = False - if hasattr(request, "spaces_between_special_tokens"): - request.spaces_between_special_tokens = False - return request - - def is_reasoning_end(self, input_ids: Sequence[int]) -> bool: - if not self._thinking_enabled: - return True - # Reasoning has ended only if the *most recent* think block is closed: - # the last close marker must come after the last open marker. A plain - # "a close marker exists anywhere" check false-positives in multi-turn / - # agent continuations, where the chat template keeps a prior turn's - # think channel (with its <|close|>think<|sep|>) in the prompt while the - # current turn is still reasoning (its <|open|>think<|sep|> is the newest - # marker). A missing open marker (e.g. it was consumed as the generation - # prefix) means a close marker alone ends reasoning, which is what - # "close is the newest marker" already encodes. - return ( - _newest_marker(input_ids, self._think_close_ids, self._think_open_ids) == 0 - ) - - def is_reasoning_end_streaming( - self, input_ids: Sequence[int], delta_ids: Iterable[int] - ) -> bool: - """Reasoning-end check for a single decode step. - - The engine calls this once per structured-output request per decode step - while the request is still inside the think channel, so it only has to - look at the tokens generated this step, plus ``len(marker) - 1`` tokens - of context in case a marker straddles the step boundary. - - The inherited default re-runs the full-sequence ``is_reasoning_end``, - which makes the scheduler O(context) per request per step and starves the - GPU on long agentic contexts. - """ - if not self._thinking_enabled: - return True - delta = list(delta_ids) - if not delta: - return False - carry = max(len(self._think_close_ids), len(self._think_open_ids)) - 1 - head = len(input_ids) - len(delta) - window = list(input_ids[max(0, head - carry) : head]) + delta - return _newest_marker(window, self._think_close_ids, self._think_open_ids) == 0 - - def _extract_content_ids(self, input_ids: list[int]) -> list[int]: - if not self._thinking_enabled: - return input_ids - idx = _subseq_index(input_ids, self._think_close_ids) - if idx == -1: - return [] # still reasoning - return input_ids[idx + len(self._think_close_ids) :] - - def extract_content_ids(self, input_ids: list[int]) -> list[int]: - cached_delta_ids = self._last_streaming_delta_token_ids - cached_content_ids = self._last_streaming_content_token_ids - self._last_streaming_delta_token_ids = None - self._last_streaming_content_token_ids = None - if cached_delta_ids == tuple(input_ids) and cached_content_ids is not None: - return cached_content_ids - return self._extract_content_ids(input_ids) - - def count_reasoning_tokens(self, token_ids: Sequence[int]) -> int: - if not self._thinking_enabled: - return 0 - think_open = self._think_open_ids - think_close = self._think_close_ids - response_open = self._response_open_ids - first_ids = {m[0] for m in (think_open, think_close, response_open) if m} - n = len(token_ids) - count = 0 - in_reasoning = True - seen_open = False - i = 0 - while i < n: - head = token_ids[i] - if head in first_ids: - if i + len(think_open) <= n and _match_at(token_ids, i, think_open): - in_reasoning = True - seen_open = True - i += len(think_open) - continue - if i + len(think_close) <= n and _match_at(token_ids, i, think_close): - in_reasoning = False - i += len(think_close) - continue - if ( - not seen_open - and i + len(response_open) <= n - and _match_at(token_ids, i, response_open) - ): - in_reasoning = False - i += len(response_open) - continue - if in_reasoning: - count += 1 - i += 1 - return count - - def _strip_content_wrapper(self, text: str) -> str: - """Strip ``<|open|>response<|sep|>…<|close|>response<|sep|>`` wrapper and - ``<|close|>message<|sep|>`` from *text*. - - When Kimi K3 tool parsing is active it needs the raw XTML - ``response`` + ``tools`` channels. Otherwise the reasoning parser cleans - up the response wrapper itself so API users do not see XTML markers. - """ - # Try structured unwrap first: extract body of response channel - m_ro = self._response_open_re.search(text) - m_rc = self._response_close_re.search(text, m_ro.end() if m_ro else 0) - if m_ro is not None and m_rc is not None: - text = text[m_ro.end() : m_rc.start()] - elif m_ro is not None: - # response opened but not closed (truncated) - text = text[m_ro.end() :] - else: - # No response wrapper — strip stray markers if present - text = self._response_open_re.sub("", text) - text = self._response_close_re.sub("", text) - text = self._message_close_re.sub("", text) - return text - - @staticmethod - def _should_preserve_tool_channels( - request: "ChatCompletionRequest | ResponsesRequest", - ) -> bool: - return bool(getattr(request, "tools", None)) and ( - getattr(request, "tool_choice", None) != "none" - ) - - def _content_after_reasoning( - self, - text: str, - request: "ChatCompletionRequest | ResponsesRequest", - ) -> str | None: - if self._should_preserve_tool_channels(request): - return text or None - return self._strip_content_wrapper(text) or None - - def extract_reasoning( - self, model_output: str, request: "ChatCompletionRequest | ResponsesRequest" - ) -> tuple[str | None, str | None]: - """Split full text into ``(reasoning, rest)`` for the non-streaming path. - - Handles four shapes: - * response opener without think markers -> no think channel; all content - * open marker present -> reasoning starts after ``<|open|>think<|sep|>`` - * open marker absent but a close marker exists (gen-prefix consumed - the open) -> reasoning starts at offset 0 - * neither marker present -> truncated reasoning after a consumed - generation prefix - ``rest`` is whatever follows the close marker, fed on to the tool parser. - """ - if not self._thinking_enabled: - return None, self._content_after_reasoning(model_output, request) - - m_open = self._think_open_re.search(model_output) - # reasoning content begins right after think-open (or at start if the - # open marker was already consumed as a generation prefix) - content_start = m_open.end() if m_open is not None else 0 - - m_close = self._think_close_re.search(model_output, content_start) - if ( - m_open is None - and m_close is None - and self._response_open_re.search(model_output) is not None - ): - return None, self._content_after_reasoning(model_output, request) - if m_close is not None: - reasoning = model_output[content_start : m_close.start()] - rest = model_output[m_close.end() :] - return (reasoning or None, self._content_after_reasoning(rest, request)) - # think not closed -> still reasoning, no content yet - return (model_output[content_start:] or None, None) - - def _reasoning_text_ready_to_emit(self, text: str) -> str: - """Return the reasoning prefix that is safe to stream now. - - Work from accumulated text, not from the current delta alone. That turns - split-open and split-close handling into the same prefix-diff problem. - - Example 1: split think-open marker. - chunks: ``<|open|>`` / ``think`` / ``<|sep|>reasoning`` - current text after chunk 1: ``<|open|>`` -> emit ``""`` - current text after chunk 2: ``<|open|>think`` -> emit ``""`` - current text after chunk 3: ``<|open|>think<|sep|>reasoning`` - -> emit ``reasoning`` - - Example 2: split think-close marker. - chunks: ``reasoning`` / ``<|close|>`` / ``think<|sep|>...`` - after ``<|close|>``, the suffix is only a partial close marker, so the - sendable reasoning is still ``reasoning`` and the delta is empty. Once - ``think<|sep|>`` arrives, the close branch hands the following response - or tools text to the downstream parser. - """ - m_open = self._think_open_re.search(text) - if m_open is not None: - text = text[m_open.end() :] - overlap = 0 - for marker in (self._think_open, self._think_close): - max_check = min(len(marker) - 1, len(text)) - for n in range(max_check, 0, -1): - if text.endswith(marker[:n]): - overlap = max(overlap, n) - break - return text[:-overlap] if overlap else text - - def _content_ready_to_emit(self, text: str) -> str: - """Return the content prefix that is safe to stream now. - - Mirrors ``_reasoning_text_ready_to_emit`` but for the post-reasoning - content phase. Strips the ``<|open|>response<|sep|>`` prefix, holds back - any partial marker suffix, and removes complete - ``<|close|>response<|sep|>`` / ``<|close|>message<|sep|>`` markers. - """ - # Strip response-open prefix - m_open = self._response_open_re.search(text) - if m_open is not None: - text = text[m_open.end() :] - - # Remove complete close/message markers - text = self._response_close_re.sub("", text) - text = self._message_close_re.sub("", text) - - # Hold back partial markers at the end - overlap = 0 - for marker in ( - self._response_open, - self._response_close, - self._message_close, - ): - max_check = min(len(marker) - 1, len(text)) - for n in range(max_check, 0, -1): - if text.endswith(marker[:n]): - overlap = max(overlap, n) - break - return text[:-overlap] if overlap else text - - def strip_content_streaming( - self, - previous_text: str, - current_text: str, - ) -> DeltaMessage | None: - """Strip XTML content wrappers from streaming deltas after reasoning. - - Called by KimiK3Parser when no tool parser is configured, so the - reasoning parser handles ``<|open|>response<|sep|>`` / - ``<|close|>response<|sep|>`` / ``<|close|>message<|sep|>`` stripping itself. - - Works from accumulated text (``previous_text`` / ``current_text`` - already contain only post-reasoning content). - """ - current_safe = self._content_ready_to_emit(current_text) - previous_safe = self._content_ready_to_emit(previous_text) - if current_safe.startswith(previous_safe): - delta = current_safe[len(previous_safe) :] - else: - delta = current_safe - return DeltaMessage(content=delta) if delta else None - - def extract_reasoning_streaming( - self, - previous_text: str, - current_text: str, - delta_text: str, - previous_token_ids: Sequence[int], - current_token_ids: Sequence[int], - delta_token_ids: Sequence[int], - ) -> DeltaMessage | None: - self._last_streaming_delta_token_ids = None - self._last_streaming_content_token_ids = None - if not self._thinking_enabled: - return DeltaMessage(content=delta_text) - - # reasoning already ended -> downstream content - if self._think_close_re.search(previous_text): - return DeltaMessage(content=delta_text) - - # the close marker completes within this delta's accumulated text: - # split the buffer at the close marker into reasoning vs trailing content. - m_close = self._think_close_re.search(current_text) - if m_close is not None: - self._last_streaming_delta_token_ids = tuple(delta_token_ids) - self._last_streaming_content_token_ids = self._extract_content_ids( - list(current_token_ids) - ) - m_open = self._think_open_re.search(current_text) - r_start = m_open.end() if m_open is not None else 0 - reasoning = current_text[r_start : m_close.start()] - already_sent = self._reasoning_text_ready_to_emit(previous_text) - if reasoning.startswith(already_sent): - reasoning_delta = reasoning[len(already_sent) :] - else: - reasoning_delta = reasoning - content = current_text[m_close.end() :] - return DeltaMessage( - reasoning=reasoning_delta or None, - content=content or None, - ) - - current_reasoning = self._reasoning_text_ready_to_emit(current_text) - previous_reasoning = self._reasoning_text_ready_to_emit(previous_text) - if current_reasoning.startswith(previous_reasoning): - reasoning_delta = current_reasoning[len(previous_reasoning) :] - else: - reasoning_delta = current_reasoning - if not reasoning_delta: - return None - return DeltaMessage(reasoning=reasoning_delta) - - # Backward-compatible aliases for existing unit tests and downstream users - # that still call the pre-split method names. - extract_reasoning_content = extract_reasoning - extract_reasoning_content_streaming = extract_reasoning_streaming diff --git a/vllm/reasoning/ling3_reasoning_parser.py b/vllm/reasoning/ling3_reasoning_parser.py deleted file mode 100644 index 43105585c8..0000000000 --- a/vllm/reasoning/ling3_reasoning_parser.py +++ /dev/null @@ -1,6 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from vllm.parser.ling3 import Ling3ParserReasoningAdapter - -__all__ = ["Ling3ParserReasoningAdapter"] diff --git a/vllm/reasoning/mimo_engine_reasoning_parser.py b/vllm/reasoning/mimo_engine_reasoning_parser.py deleted file mode 100644 index d07cde9278..0000000000 --- a/vllm/reasoning/mimo_engine_reasoning_parser.py +++ /dev/null @@ -1,6 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from vllm.parser.engine.registered_adapters import MiMoParserReasoningAdapter - -__all__ = ["MiMoParserReasoningAdapter"] diff --git a/vllm/reasoning/minicpmv_reasoning_parser.py b/vllm/reasoning/minicpmv_reasoning_parser.py deleted file mode 100644 index 402c0524ca..0000000000 --- a/vllm/reasoning/minicpmv_reasoning_parser.py +++ /dev/null @@ -1,16 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from vllm.parser.engine.adapters import ParserEngineReasoningAdapter -from vllm.parser.minicpmv import MiniCPMVParser - - -class MiniCPMVParserReasoningAdapter(ParserEngineReasoningAdapter): - # Reasoning-only on purpose: this family registers no tool parser, because - # MiniCPM-V emits the Qwen3-Coder XML syntax verbatim and `qwen3_coder` - # already covers it. ``make_adapters`` only builds reasoning/tool pairs, so - # the two halves are declared explicitly here. - _parser_engine_cls = MiniCPMVParser - - -__all__ = ["MiniCPMVParserReasoningAdapter"] diff --git a/vllm/reasoning/minimax_m2_reasoning_parser.py b/vllm/reasoning/minimax_m2_reasoning_parser.py deleted file mode 100644 index 8fa5b45d58..0000000000 --- a/vllm/reasoning/minimax_m2_reasoning_parser.py +++ /dev/null @@ -1,61 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from collections.abc import Sequence -from typing import TYPE_CHECKING - -from vllm.entrypoints.generate.base.protocol import DeltaMessage -from vllm.parser.engine.registered_adapters import MinimaxM2ParserReasoningAdapter -from vllm.reasoning.abs_reasoning_parsers import ReasoningParser -from vllm.tokenizers import TokenizerLike - -if TYPE_CHECKING: - from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest - from vllm.entrypoints.openai.responses.protocol import ResponsesRequest - - -class MiniMaxM2ReasoningParser(MinimaxM2ParserReasoningAdapter): # type: ignore[valid-type, misc] - """Reasoning parser for MiniMax M2 model. - - MiniMax M2 models don't generate start token, only end - token. All content before is reasoning, content after is the - actual response. - """ - - -class MiniMaxM2AppendThinkReasoningParser(ReasoningParser): - """Reasoning parser for MiniMax M2 model.""" - - def __init__(self, tokenizer: TokenizerLike, *args, **kwargs): - super().__init__(tokenizer, *args, **kwargs) - self.end_token_id = self.vocab.get("") - self.start_token_id = self.vocab.get("") - - def is_reasoning_end(self, input_ids: Sequence[int]) -> bool: - end_token_id = self.end_token_id - start_token_id = self.start_token_id - for input_id in reversed(input_ids): - if input_id in (end_token_id, start_token_id): - return input_id == end_token_id - return False - - def extract_content_ids(self, input_ids: list[int]) -> list[int]: - return input_ids - - def extract_reasoning_streaming( - self, - previous_text: str, - current_text: str, - delta_text: str, - previous_token_ids: Sequence[int], - current_token_ids: Sequence[int], - delta_token_ids: Sequence[int], - ) -> DeltaMessage | None: - if len(previous_token_ids) == 0: - delta_text = "" + delta_text - return DeltaMessage(content=delta_text) - - def extract_reasoning( - self, model_output: str, request: "ChatCompletionRequest | ResponsesRequest" - ) -> tuple[str | None, str | None]: - return None, "" + model_output diff --git a/vllm/reasoning/minimax_m3_reasoning_parser.py b/vllm/reasoning/minimax_m3_reasoning_parser.py deleted file mode 100644 index d49f9311f4..0000000000 --- a/vllm/reasoning/minimax_m3_reasoning_parser.py +++ /dev/null @@ -1,320 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from collections.abc import Iterable, Sequence -from typing import TYPE_CHECKING - -from vllm.entrypoints.generate.base.protocol import DeltaMessage -from vllm.reasoning.basic_parsers import BaseThinkingReasoningParser - -if TYPE_CHECKING: - from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest - from vllm.entrypoints.openai.responses.protocol import ResponsesRequest - - -class MiniMaxM3ReasoningParser(BaseThinkingReasoningParser): - """Reasoning parser for MiniMax M3 explicit thinking blocks. - - MiniMax M3 emits reasoning as: - - reasoning textassistant content - - The M3 tokenizer exposes both markers as complete vocabulary entries, but - generated marker text may be tokenized into smaller pieces. The streaming - parser therefore uses text markers for extraction instead of relying on the - single vocabulary IDs. The chat template may also prefill the start marker - when ``thinking_mode="enabled"``, so generated text can begin directly - inside a reasoning block without emitting ```` again. - """ - - @property - def start_token(self) -> str: - return "" - - @property - def end_token(self) -> str: - return "" - - def __init__(self, tokenizer, *args, **kwargs): - super().__init__(tokenizer, *args, **kwargs) - self._start_token_ids = self._encode_marker(self.start_token) - self._end_token_ids = self._encode_marker(self.end_token) - chat_kwargs = kwargs.get("chat_template_kwargs", {}) or {} - self._initial_in_reasoning = chat_kwargs.get("thinking_mode") == "enabled" - self._reasoning_ended_streaming = False - self._reasoning_active_streaming = self._initial_in_reasoning - self._pending_marker_streaming = False - self._last_streaming_delta_token_ids: tuple[int, ...] | None = None - self._last_streaming_content_token_ids: list[int] | None = None - - def _encode_text(self, text: str) -> list[int]: - try: - return list(self.model_tokenizer.encode(text, add_special_tokens=False)) - except TypeError: - return list(self.model_tokenizer.encode(text)) - - def _encode_marker(self, marker: str) -> tuple[int, ...]: - return tuple(self._encode_text(marker)) - - def _decode_text(self, token_ids: Sequence[int]) -> str: - try: - return self.model_tokenizer.decode( - list(token_ids), skip_special_tokens=False - ) - except TypeError: - return self.model_tokenizer.decode(list(token_ids)) - - def _content_suffix_token_ids( - self, - delta_text: str, - delta_token_ids: Sequence[int], - content: str | None, - ) -> list[int]: - if content is None: - return [] - if content == delta_text: - return list(delta_token_ids) - if delta_text.endswith(content): - prefix_text = delta_text[: len(delta_text) - len(content)] - for index in range(len(delta_token_ids) + 1): - if self._decode_text(delta_token_ids[:index]) == prefix_text: - return list(delta_token_ids[index:]) - return self._encode_text(content) - - @staticmethod - def _contains_token_sequence( - token_ids: Sequence[int], marker_ids: Sequence[int] - ) -> bool: - if not marker_ids or len(marker_ids) > len(token_ids): - return False - marker_len = len(marker_ids) - return any( - tuple(token_ids[i : i + marker_len]) == tuple(marker_ids) - for i in range(len(token_ids) - marker_len + 1) - ) - - @staticmethod - def _rfind_token_sequence( - token_ids: Sequence[int], marker_ids: Sequence[int] - ) -> int: - if not marker_ids or len(marker_ids) > len(token_ids): - return -1 - marker_len = len(marker_ids) - for i in range(len(token_ids) - marker_len, -1, -1): - if tuple(token_ids[i : i + marker_len]) == tuple(marker_ids): - return i - return -1 - - @staticmethod - def _ends_with_token_sequence_prefix( - token_ids: Sequence[int], marker_ids: Sequence[int] - ) -> bool: - if not marker_ids: - return False - max_len = min(len(token_ids), len(marker_ids) - 1) - for prefix_len in range(max_len, 0, -1): - if tuple(token_ids[-prefix_len:]) == tuple(marker_ids[:prefix_len]): - return True - return False - - @staticmethod - def _strip_partial_marker_suffix(text: str, marker: str) -> str: - max_len = min(len(text), len(marker) - 1) - for suffix_len in range(max_len, 0, -1): - if marker.startswith(text[-suffix_len:]): - return text[:-suffix_len] - return text - - @staticmethod - def _visible_delta(previous: str | None, current: str | None) -> str | None: - if not current: - return None - if not previous: - return current - if current.startswith(previous): - delta = current[len(previous) :] - return delta or None - return current - - def _visible_segments(self, text: str) -> tuple[str | None, str | None]: - if not text: - return None, None - - if not self._initial_in_reasoning: - if self.end_token.startswith(text) and len(text) < len(self.end_token): - return None, None - if text.startswith(self.end_token): - text = text[len(self.end_token) :] - if not text: - return None, None - - if self._initial_in_reasoning and self.start_token not in text: - reasoning, end, content = text.partition(self.end_token) - if end: - return reasoning or None, content or None - reasoning = self._strip_partial_marker_suffix(reasoning, self.end_token) - return reasoning or None, None - - if self.start_token not in text: - content = self._strip_partial_marker_suffix(text, self.start_token) - return None, content or None - - content_before, _, after_start = text.partition(self.start_token) - reasoning, end, content_after = after_start.partition(self.end_token) - if end: - return reasoning or None, (content_before + content_after) or None - - reasoning = self._strip_partial_marker_suffix(reasoning, self.end_token) - return reasoning or None, content_before or None - - def extract_reasoning( - self, - model_output: str, - request: "ChatCompletionRequest | ResponsesRequest", - ) -> tuple[str | None, str | None]: - # MiniMax M3 can start a response with a stray closer. Drop that first - # token only; later unmatched closers stay visible as content. - if not self._initial_in_reasoning and model_output.startswith(self.end_token): - content = model_output[len(self.end_token) :] - return None, content or None - - if self._initial_in_reasoning and self.start_token not in model_output: - reasoning, end, content = model_output.partition(self.end_token) - if not end: - return model_output, None - return reasoning, content or None - - if self.start_token not in model_output: - return None, model_output - - content_before, _, after_start = model_output.partition(self.start_token) - reasoning, end, content_after = after_start.partition(self.end_token) - if not end: - return reasoning, content_before or None - - return reasoning, (content_before + content_after) or None - - def is_reasoning_end_streaming( - self, input_ids: Sequence[int], delta_ids: Iterable[int] - ) -> bool: - if self._reasoning_ended_streaming: - return True - - if self._reasoning_active_streaming or self._pending_marker_streaming: - return False - - delta_ids = tuple(delta_ids) - if self._contains_token_sequence(delta_ids, self._end_token_ids): - return True - if self._contains_token_sequence(input_ids, self._end_token_ids): - return True - if self._initial_in_reasoning: - return False - if self._ends_with_token_sequence_prefix(input_ids, self._start_token_ids): - return False - if self._ends_with_token_sequence_prefix(input_ids, self._end_token_ids): - return False - if not self._contains_token_sequence(input_ids, self._start_token_ids): - return bool(input_ids) - return False - - def extract_content_ids(self, input_ids: list[int]) -> list[int]: - if ( - self._last_streaming_delta_token_ids == tuple(input_ids) - and self._last_streaming_content_token_ids is not None - ): - content_ids = self._last_streaming_content_token_ids - self._last_streaming_delta_token_ids = None - self._last_streaming_content_token_ids = None - return list(content_ids) - - end_index = self._rfind_token_sequence(input_ids, self._end_token_ids) - if end_index >= 0: - return input_ids[end_index + len(self._end_token_ids) :] - - has_start = self._contains_token_sequence(input_ids, self._start_token_ids) - if self._initial_in_reasoning and not has_start: - return [] - - if not has_start: - return input_ids - return [] - - def extract_reasoning_streaming( - self, - previous_text: str, - current_text: str, - delta_text: str, - previous_token_ids: Sequence[int], - current_token_ids: Sequence[int], - delta_token_ids: Sequence[int], - ) -> DeltaMessage | None: - if not delta_text: - return None - - if not previous_text: - self._reasoning_ended_streaming = False - self._reasoning_active_streaming = self._initial_in_reasoning - self._pending_marker_streaming = False - self._last_streaming_delta_token_ids = None - self._last_streaming_content_token_ids = None - previous_reasoning, previous_content = self._visible_segments(previous_text) - current_reasoning, current_content = self._visible_segments(current_text) - if self.end_token in current_text or current_content is not None: - self._reasoning_ended_streaming = True - self._reasoning_active_streaming = False - self._pending_marker_streaming = False - else: - self._last_streaming_delta_token_ids = None - self._last_streaming_content_token_ids = None - self._reasoning_active_streaming = ( - self._initial_in_reasoning - or self.start_token in current_text - or current_reasoning is not None - ) - self._pending_marker_streaming = not self._reasoning_active_streaming and ( - self.start_token.startswith(current_text) - or self.end_token.startswith(current_text) - ) - reasoning = self._visible_delta(previous_reasoning, current_reasoning) - content = self._visible_delta(previous_content, current_content) - if self._reasoning_ended_streaming: - self._last_streaming_delta_token_ids = tuple(delta_token_ids) - self._last_streaming_content_token_ids = self._content_suffix_token_ids( - delta_text, delta_token_ids, content - ) - if reasoning is None and content is None: - return None - return DeltaMessage(reasoning=reasoning, content=content) - - def count_reasoning_tokens(self, token_ids: Sequence[int]) -> int: - count = 0 - depth = 1 if self._initial_in_reasoning else 0 - i = 0 - while i < len(token_ids): - if tuple(token_ids[i : i + len(self._start_token_ids)]) == ( - self._start_token_ids - ): - depth += 1 - i += len(self._start_token_ids) - continue - if tuple(token_ids[i : i + len(self._end_token_ids)]) == ( - self._end_token_ids - ): - if depth > 0: - depth -= 1 - i += len(self._end_token_ids) - continue - if depth > 0: - count += 1 - i += 1 - return count - - def is_reasoning_end(self, input_ids: Sequence[int]) -> bool: - start_index = self._rfind_token_sequence(input_ids, self._start_token_ids) - end_index = self._rfind_token_sequence(input_ids, self._end_token_ids) - if end_index < 0: - return False - if start_index < 0: - return True - return end_index > start_index diff --git a/vllm/reasoning/mistral_reasoning_parser.py b/vllm/reasoning/mistral_reasoning_parser.py deleted file mode 100644 index 6adbb08cb5..0000000000 --- a/vllm/reasoning/mistral_reasoning_parser.py +++ /dev/null @@ -1,6 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from vllm.parser.engine.registered_adapters import MistralParserReasoningAdapter - -__all__ = ["MistralParserReasoningAdapter"] diff --git a/vllm/reasoning/muse_glimmer_reasoning_parser.py b/vllm/reasoning/muse_glimmer_reasoning_parser.py deleted file mode 100644 index 5991eb4d2d..0000000000 --- a/vllm/reasoning/muse_glimmer_reasoning_parser.py +++ /dev/null @@ -1,327 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Reasoning-content parser for MuseGlimmer. -Port of the ``reasoning_content`` rule from the HuggingFace MuseGlimmer -``MUSE_GLIMMER_RESPONSE_SCHEMA`` (synced with internal master). MuseGlimmer emits -chain-of-thought in ``to=self`` channels delimited by ``<|message|>`` ... ``<|eom|>``: - to=self<|message|>...reasoning...<|eom|> -A turn may contain several ``to=self`` blocks interleaved with tool calls, and a -tool call or final answer follows in its own channel. -Because MuseGlimmer's framing markers (``<|message|>``, ``<|eom|>``) are not guaranteed -to be single vocab tokens across every checkpoint's tokenizer, this parser works -on the decoded text with regexes rather than the single start/end-token base class. -Usage: ``--reasoning-parser muse_glimmer`` -""" - -from __future__ import annotations - -from collections.abc import Iterable, Sequence - -import regex as re - -from vllm.entrypoints.generate.base.protocol import DeltaMessage -from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest -from vllm.entrypoints.openai.responses.protocol import ResponsesRequest -from vllm.reasoning.abs_reasoning_parsers import ReasoningParser - -_EOM = "<|eom|>" -_EOT = "<|eot|>" -_FUNCTION_CALLS_OPEN = "" -_REASONING_OPEN = "to=self<|message|>" -_ASSISTANT_TURN_OPEN = "<|start|>assistant" -# A channel header: ``to=<|message|>`` where recipient is ``self`` -# (reasoning), ``user`` (final answer) or ``[.]`` (tool call). -_CHANNEL_HEADER_RE = re.compile(r"to=(?P[^\s<]+)<\|message\|>") -_HEADER_PAT = r"to=[^\s<]+<\|message\|>" -# Collapse the gap between reasoning blocks so multiple to=self spans join. -_COLLAPSE_RE = re.compile( - r"<\|eom\|>(?:(?!to=self<\|message\|>).)*?to=self<\|message\|>", re.DOTALL -) -_REASONING_RE = re.compile(r"to=self<\|message\|>(.*?)<\|eom\|>", re.DOTALL) -_CONTENT_RE = re.compile( - r"to=user<\|message\|>(.*?)(?=<\|eot\|>|<\|eom\|>|$)", re.DOTALL -) -# Strip a CLOSED reasoning span (header .. <|eom|>). -_STRIP_REASONING_RE = re.compile( - r"(?:<\|start\|>assistant\s*)?to=self<\|message\|>.*?<\|eom\|>", re.DOTALL -) -# An UNTERMINATED trailing reasoning span. The model sometimes leaves the -# analysis channel WITHOUT emitting <|eom|>, writing a bare -# ``to=<|message|>`` header instead (observed deterministically for a call -# with EMPTY arguments on a tool that has optional parameters; reproduced on -# other engines too, so it is a model-side defect, not engine-specific). -# -# These two patterns MUST therefore stop at the next channel header rather than -# running to end-of-text. An unbounded ``...$`` version consumes the real tool -# call along with the reasoning: `is_reasoning_end` then never fires, the parser -# never leaves the reasoning phase, the tool parser is never invoked, and the -# entire generation is dropped (empty reasoning, empty content, no tool call). -_STRIP_OPEN_REASONING_RE = re.compile( - r"(?:<\|start\|>assistant\s*)?to=self<\|message\|>" - r"(?:(?!<\|eom\|>)(?!" + _HEADER_PAT + r").)*" - r"(?=" + _HEADER_PAT + r"|$)", - re.DOTALL, -) -_OPEN_REASONING_RE = re.compile( - r"to=self<\|message\|>((?:(?!<\|eom\|>)(?!" + _HEADER_PAT + r").)*)" - r"(?=" + _HEADER_PAT + r"|$)", - re.DOTALL, -) -# Markers whose PREFIX could appear at the tail of an OPEN (still-streaming) body. -_HOLDBACK_MARKERS = (_EOM, _EOT, "<|start|>", "<|message|>") -# A trailing fragment that could still grow into a channel header (" t", " to", -# " to=", " to=skill"). Without this the recipient name leaks into reasoning and -# then has to be un-emitted once ``<|message|>`` arrives. -_OPEN_TAIL_HEADER_RE = re.compile(r"[\s](?:t|to|to=[^\s<]*)$") - - -def _current_assistant_turn(text: str) -> str: - """Return only the text generated in the current assistant turn. - ``is_reasoning_end`` is evaluated on the PROMPT token-ids at stream start, - and an MuseGlimmer prompt legitimately contains ATEM markers (``render_tool_defs`` - writes a literal ```` example into the system message, - and prior assistant turns may carry real tool calls). Anchoring on the last - channel-open keeps prompt text from deciding the phase. - """ - idx = text.rfind(_ASSISTANT_TURN_OPEN) - return text[idx + len(_ASSISTANT_TURN_OPEN) :] if idx != -1 else text - - -def _trim_open_body(body: str) -> str: - """Hold back any tail of a still-growing body that could still be framing. - Iterated to a fixpoint because the two cases compose: `" to=skill<"` needs - the partial-marker trim (``<``) before the partial-header trim can see - `" to=skill"`. Trimming only once leaks the recipient name as reasoning. - """ - while True: - trimmed = body - for marker in _HOLDBACK_MARKERS: - for k in range(min(len(marker) - 1, len(trimmed)), 0, -1): - if trimmed.endswith(marker[:k]): - trimmed = trimmed[:-k] - break - else: - continue - break - header_tail = _OPEN_TAIL_HEADER_RE.search(trimmed) - if header_tail is not None: - trimmed = trimmed[: header_tail.start()] - if trimmed == body: - return body - body = trimmed - - -class MuseGlimmerReasoningParser(ReasoningParser): - def __init__(self, tokenizer, *args, **kwargs) -> None: - super().__init__(tokenizer, *args, **kwargs) - # Cursors over what was ACTUALLY emitted. Diffing a freshly reclassified - # `previous_text` is unsafe: a classified body legitimately shrinks when - # a partial header becomes recognisable, and diffing against the shrunken - # value re-emits text that already went out. - self._emitted_reasoning: str = "" - self._emitted_content: str = "" - self._tool_handoff_done: bool = False - - def adjust_request( - self, request: ChatCompletionRequest | ResponsesRequest - ) -> ChatCompletionRequest | ResponsesRequest: - """Preserve MuseGlimmer's ATEM framing tokens in the decoded output. - vLLM's serving default is ``skip_special_tokens=True``, which strips - ``<|start|>`` / ``<|message|>`` / ``<|eom|>`` / ``<|eot|>`` before the - parsers run, collapsing reasoning into content and breaking channel - scoping. Unlike the base tool-parser hook we do NOT touch - ``structured_outputs`` -- MuseGlimmer emits native ATEM, not JSON. - """ - request.skip_special_tokens = False - return request - - def is_reasoning_end(self, input_ids: Sequence[int]) -> bool: - """Whether the model has left reasoning and opened a TOOL channel. - A ``to=user`` answer is NOT a reason to leave the reasoning phase -- this - parser surfaces that content itself. Only a real tool channel switches - the ``DelegatingParser`` phase machine over to the tool parser. - Both closed and unterminated reasoning spans are stripped before the - check, so an ```` the model merely echoes inside its CoT - never flips the phase. - """ - try: - text = self.model_tokenizer.decode(input_ids) - except Exception: - return False - remainder = self._tool_channel_remainder(text) - return _FUNCTION_CALLS_OPEN in remainder or " bool: - return self.is_reasoning_end(list(input_ids)) - - def extract_content_ids(self, input_ids: list[int]) -> list[int]: - # Content-id slicing is unreliable for multi-token markers; the serving - # path uses extract_reasoning() for the final split. - return [] - - @classmethod - def _scoped_turn(cls, text: str) -> str: - """Current assistant turn with reasoning spans removed.""" - scoped = _current_assistant_turn(text) - scoped = _STRIP_REASONING_RE.sub("", scoped) - return _STRIP_OPEN_REASONING_RE.sub("", scoped) - - @classmethod - def _tool_channel_remainder(cls, text: str) -> str: - """Text from the first tool-channel header onward, framing INCLUDED. - ``DelegatingParser.parse_delta`` rebuilds ``current_text`` from whatever - this parser returns as ``.content`` on the transition delta and commits - it; anything not returned is destroyed. It must start AT the - ``to=<|message|>`` header -- handing over the text after the header - loses the recipient, and the tool parser then sees a bare ``<|message|>``, - classifies it as the content channel, and leaks the ATEM markup. - """ - scoped = cls._scoped_turn(text) - for match in _CHANNEL_HEADER_RE.finditer(scoped): - if match.group("recipient") not in ("self", "user"): - return scoped[match.start() :] - return "" - - @staticmethod - def _classify_bodies(text: str) -> tuple[str, str]: - """Split ``text`` into (reasoning_body, content_body), channel-aware. - Framing markers and tool channels contribute nothing -- the tool parser - owns those. A body ends at ``<|eom|>`` / ``<|eot|>``, at the next channel - header, or at end-of-text (an OPEN body, which is held back). - """ - reasoning_parts: list[str] = [] - content_parts: list[str] = [] - pos = 0 - n = len(text) - while pos < n: - match = _CHANNEL_HEADER_RE.search(text, pos) - if not match: - break - recipient = match.group("recipient") - body_start = match.end() - eom = text.find(_EOM, body_start) - eot = text.find(_EOT, body_start) - terminators = [p for p in (eom, eot) if p != -1] - next_header = _CHANNEL_HEADER_RE.search(text, body_start) - if next_header is not None: - terminators.append(next_header.start()) - body_end = min(terminators) if terminators else n - body = text[body_start:body_end] - if not terminators: - body = _trim_open_body(body) - if recipient == "self": - reasoning_parts.append(body) - elif ( - # Never surface tool XML echoed into a user channel. - recipient == "user" - and _FUNCTION_CALLS_OPEN not in body - and " str | None: - """Promote un-surfaced content when the stream ends mid-reasoning. - ``DelegatingParser.finalize_generation`` calls this when - ``reasoning_ended`` is still False. Returns only the channel-classified - ``to=user`` body, and only the portion not already streamed. - """ - _, content_body = self._classify_bodies(previous_text) - remainder = content_body[len(self._emitted_content) :] - return remainder or None - - def extract_reasoning( - self, - model_output: str, - request: ChatCompletionRequest | ResponsesRequest, - ) -> tuple[str | None, str | None]: - collapsed = _COLLAPSE_RE.sub("\n", model_output) - matches = _REASONING_RE.findall(collapsed) - reasoning = "\n".join(matches) if matches else None - # Truncation fallback: generation stopped inside a to=self block, so - # there is no closing <|eom|>. Bounded at the next channel header so a - # real tool call that follows a header-less channel switch is not - # absorbed into the reasoning field. - open_match = _OPEN_REASONING_RE.search(model_output) - if open_match and open_match.group(1): - partial = open_match.group(1) - reasoning = f"{reasoning}\n{partial}" if reasoning else partial - # Content is everything that is not a reasoning block. In a - # reasoning+tool-call turn there is no to=user answer, but the tool - # channels MUST be forwarded -- the unified parser runs the tool parser - # on this returned `content`, not on the original model_output. - remainder = _STRIP_REASONING_RE.sub("", model_output) - remainder = _STRIP_OPEN_REASONING_RE.sub("", remainder) - if " DeltaMessage | None: - """Channel-aware streaming split of reasoning vs content. - Classifies the full ``current_text`` and emits only what has not been - emitted yet, so no framing token is ever surfaced and a delta straddling - a channel boundary only contributes the portion inside a real body. - """ - curr_reason, curr_content = self._classify_bodies(current_text) - reasoning_delta = "" - if curr_reason.startswith(self._emitted_reasoning) and len(curr_reason) > len( - self._emitted_reasoning - ): - reasoning_delta = curr_reason[len(self._emitted_reasoning) :] - self._emitted_reasoning = curr_reason - content_delta = "" - if curr_content.startswith(self._emitted_content) and len(curr_content) > len( - self._emitted_content - ): - content_delta = curr_content[len(self._emitted_content) :] - self._emitted_content = curr_content - # Hand the tool channel to the tool parser exactly once, starting at its - # header. parse_delta discards anything not returned here. - # - # This MUST fire on the same delta where is_reasoning_end() flips, i.e. - # only once the tool channel actually contains ATEM. Emitting it earlier - # -- when only the bare `to=<|message|>` header has arrived -- keeps - # the parser in the reasoning phase, so parse_delta never replaces this - # DeltaMessage with the tool parser's and the header is delivered to the - # client as visible content. - handoff = "" - if not self._tool_handoff_done: - remainder = self._tool_channel_remainder(current_text) - if _FUNCTION_CALLS_OPEN in remainder or " tuple[Indices | None, Indices | None]: - """Find the longest overlap where the end of string a matches the start - of string b. - - Args: - a: First string - b: Second string - - Returns: - Tuple of IndicesTuples representing the overlapping portions in each - string, or a tuple of None if no overlap exists - - """ - # swap so a is always the shorter string - a, b, swap = (a, b, False) if len(a) < len(b) else (b, a, True) - - # first check: is a fully contained in b? - if a in b: - ind_a = Indices(0, len(a)) - ind_b = Indices(b.index(a), b.index(a) + len(a)) - return (ind_b, ind_a) if swap else (ind_a, ind_b) - - # second check: does the end of a overlap with the - # beginning of b? - for i in range(len(a) - 1, 0, -1): - if a[-i:] == b[:i]: - ind_a = Indices(len(a) - i, len(a)) - ind_b = Indices(0, i) - return (ind_b, ind_a) if swap else (ind_a, ind_b) - - # third check: does the beginning of a overlap with - # the end of b? - for i in range(len(a) - 1, 0, -1): - if b[-i:] == a[:i]: - ind_a = Indices(0, i) - ind_b = Indices(len(b) - i, len(b)) - return (ind_b, ind_a) if swap else (ind_a, ind_b) - - return None, None - - -@dt.dataclass -class Olmo3ReasoningBuffer: - think_start: str = "" - think_end: str = "" - buffer: str = "" - - # we start in reasoning state to support cases where we hardcode - # as the start of the reasoning block. - # In those cases, the only token we will see is , which - # is when we switch to content state. - state: Olmo3ReasoningState = Olmo3ReasoningState.REASONING - - def process_buffer(self) -> DeltaMessage | None: - start_think_idx = self.buffer.find(self.think_start) - - if start_think_idx >= 0: - self.state = Olmo3ReasoningState.REASONING - pretext, self.buffer = ( - self.buffer[:start_think_idx], - self.buffer[start_think_idx + len(self.think_start) :], - ) - if start_think_idx > 0: - # this covers the case there's content before - # the start of the reasoning block - return DeltaMessage(content=pretext) - - end_think_idx = self.buffer.rfind(self.think_end) - - if end_think_idx >= 0: - self.state = Olmo3ReasoningState.CONTENT - pretext, self.buffer = ( - self.buffer[:end_think_idx], - self.buffer[end_think_idx + len(self.think_end) :], - ) - if end_think_idx > 0: - # this covers the case there's content before - # the end of the reasoning block - return DeltaMessage(reasoning=pretext) - - if self.state == Olmo3ReasoningState.REASONING: - # we are inside reasoning block, return and empty - # the text buffer - ( - text_buffer, - self.buffer, - ) = self.buffer, "" - return DeltaMessage(reasoning=text_buffer) - - if self.state == Olmo3ReasoningState.CONTENT: - # we are outside reasoning block, return and empty - # the text buffer - ( - text_buffer, - self.buffer, - ) = self.buffer, "" - return DeltaMessage(content=text_buffer) - - # nothing to return unless we are in reasoning or content state - return None - - def __len__(self): - # is the length of the text buffer - return len(self.buffer) - - def add_text(self, delta_text: str) -> DeltaMessage | None: - # we start by adding the delta text to the buffer - self.buffer += delta_text - - # setting this to empty before starting - delta_message: DeltaMessage | None = None - - # we start by computing the overlap between the delta_text - # and start/end of think tokens. - _, overlap_think_start = string_overlap(delta_text, self.think_start) - _, overlap_think_end = string_overlap(delta_text, self.think_end) - - partial_overlap_start = overlap_think_start is not None and len( - overlap_think_start - ) < len(self.think_start) - partial_overlap_end = overlap_think_end is not None and len( - overlap_think_end - ) < len(self.think_end) - - if ( - partial_overlap_start - and self.think_start in self.buffer - and not partial_overlap_end - ): - # we can only process the buffer if partial overlap - # is the last part of think token (thus causing - # text_buffer to contain the start of think token) - # and there are no partial overlaps with end think - delta_message = self.process_buffer() - - elif partial_overlap_end and self.think_end in self.buffer: - # same as before (partial overlap only allowed) - # if the buffer contains the end think token, - # but we don't have to check for partial overlap - # with start think token because they are handled - # by the previous condition - delta_message = self.process_buffer() - - elif partial_overlap_start or partial_overlap_end: - # in general, if there are overlaps, we don't - # process the buffer because we want to wait until - # the think token is fully completed. - return None - else: - # we process the buffer as normal - delta_message = self.process_buffer() - - return delta_message - - -class Olmo3ReasoningParser(ReasoningParser): - """Reasoning parser for Olmo 3 model. - - Olmo3ReasoningParser - - This class implements a reasoning parser specifically designed for the - Olmo 3 family of models. Olmo 3 models do not use special tokens to - indicate reasoning; rather, reasoning trace is wrapped in `` and - ``, which are tokenized using standard vocabulary entries. - Because of this, the parser operates in string space, accumulating the - characters in a buffer until it sees `` or ``. tokens - to switch modes. - - Key Features: - - For non-stream output, Recognizes and extracts reasoning (text - bracketed by `` and ``) and content (everything - after the first ``). - - For stream process, it uses a buffer to accumulate delta text, - and output progressive delta messages as soon as thinking starts - or ends. - - For reliability, some Olmo 3 models may hardcode the first - `` token is the input text (similar to Deepseek R1, - or reasoning-only Qwen models). To support such variants, the - parser can optionally work in cases where the first `` - token is missing from generation. - """ - - think_start: str = r"" - think_end: str = r"" - # is split in 3 by the pre-tokenizer, first split can be tokenized - # with an optional leading space, so there are 2 possible tokenizations - think_end_first_split: list[str] = [r"Ġ"] - # notice that the first think is optional; this allows template to - # work in cases when we hardcode a at the beginning of the - # reasoning template. - reasoning_regex: re.Pattern = re.compile( - rf"^(?:{think_start})?(?P.*?)" - rf"{think_end}(?P.*)$", - re.DOTALL, - ) - - def __init__(self, tokenizer: "TokenizerLike", *args, **kwargs): - super().__init__(tokenizer, *args, **kwargs) - self.buffer = Olmo3ReasoningBuffer( - think_start=self.think_start, think_end=self.think_end - ) - self.think_end_first_token_ids: list[int] = [ - self.vocab[token] for token in self.think_end_first_split - ] - self.think_end_rest_token_ids: list[int] = [ - self.vocab[token] for token in self.think_end_rest_split - ] - - @property - def reasoning_start_str(self) -> str: - return self.think_start - - @property - def reasoning_end_str(self) -> str: - return self.think_end - - def is_reasoning_end(self, input_ids: Sequence[int]) -> bool: - rest_ids = self.think_end_rest_token_ids - rest_len = len(rest_ids) - for i in range(len(input_ids) - rest_len, -1, -1): - if ( - list(input_ids[i + 1 : i + 1 + rest_len]) == rest_ids - and input_ids[i] in self.think_end_first_token_ids - ): - return True - return False - - def extract_content_ids(self, input_ids: list[int]) -> list[int]: - # for Olmo 3 streaming reason parsing, the stream parse - # will call first, and the same token will be called in - # is_reasoning_end and extract_content_ids - # this id is not part of content, so just return [] here. - return [] - - def extract_reasoning( - self, - model_output: str, - request: "ChatCompletionRequest | ResponsesRequest", - ) -> tuple[str | None, str | None]: - """Extract the reasoning content & content sections, respectively. - If the sequence doesn't match what we expect, i.e., the model generates - something else, all content is considered non-reasoning content. - - Args: - model_output: Output of the model to be parsed. - request: Request being - processed. - - Returns: - tuple[Optional[str], Optional[str]]: Tuple pair containing the - reasoning content and non-reasoning content. - - """ - re_match = self.reasoning_regex.match(model_output) - if re_match: - reasoning = re_match.group("reasoning") or None - content = re_match.group("content") or None - return reasoning, content - - # no reasoning content - return None, model_output - - def extract_reasoning_streaming( - self, - previous_text: str, - current_text: str, - delta_text: str, - previous_token_ids: Sequence[int], - current_token_ids: Sequence[int], - delta_token_ids: Sequence[int], - ) -> DeltaMessage | None: - """Extract content using token ID sequence state machine.""" - delta_message = self.buffer.add_text(delta_text) - if delta_message is None and self.buffer.think_end in self.buffer.buffer: - # this is a bit hacky, but, because of how the buffer is - # constructed, if the last delta_text contains characters that - # marks the end of thinking tokens, then messages in the buffer - # would never be processed because we get no other turn. To get - # around that, we check if the text buffer contains the end of - # thinking tokens, and, if so, we reprocess the buffer again. - delta_message = self.buffer.process_buffer() - - return delta_message diff --git a/vllm/reasoning/plamo3_engine_reasoning_parser.py b/vllm/reasoning/plamo3_engine_reasoning_parser.py deleted file mode 100644 index 20037fe08f..0000000000 --- a/vllm/reasoning/plamo3_engine_reasoning_parser.py +++ /dev/null @@ -1,6 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from vllm.parser.engine.registered_adapters import Plamo3ParserReasoningAdapter - -__all__ = ["Plamo3ParserReasoningAdapter"] diff --git a/vllm/reasoning/poolside_v1_reasoning_parser.py b/vllm/reasoning/poolside_v1_reasoning_parser.py deleted file mode 100644 index e854afe4bc..0000000000 --- a/vllm/reasoning/poolside_v1_reasoning_parser.py +++ /dev/null @@ -1,71 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Laguna reasoning parser. - -``DeepSeekV3ReasoningParser.is_reasoning_end`` walks the entire -token sequence backwards and returns ``True`` on the first ```` it -sees. When called on ``prompt_token_ids`` that mistakes any stray -```` in conversation history, few-shot examples or tool descriptions -for a template-injected "thinking already ended" marker. In the streaming -path (see ``vllm/entrypoints/openai/chat_completion/serving.py``, -``prompt_is_reasoning_end_arr``) that false positive short-circuits the -reasoning parser for the whole response, so any ``...`` the -model emits itself ends up in the content field instead of the reasoning -field. - -As we have more flexible templates, we instead scope -the backward search to the current assistant turn: the -walk terminates as soon as we hit the ```` start-of-message -token. A ```` in a prior user turn or few-shot example is no longer -visible. -""" - -from collections.abc import Sequence - -from transformers import PreTrainedTokenizerBase - -from vllm.reasoning.deepseek_r1_reasoning_parser import DeepSeekR1ReasoningParser -from vllm.reasoning.deepseek_v3_reasoning_parser import DeepSeekV3ReasoningParser -from vllm.reasoning.identity_reasoning_parser import IdentityReasoningParser - - -class PoolsideV1ReasoningParser(DeepSeekV3ReasoningParser): - """Drop-in replacement for ``deepseek_v3`` that tolerates ```` - tokens appearing anywhere in the prompt other than the generation prefix. - """ - - _start_of_assistant_message = "" - - def __init__(self, tokenizer: PreTrainedTokenizerBase, *args, **kwargs): - super().__init__(tokenizer, *args, **kwargs) - - if self._start_of_assistant_message not in self.vocab: - raise ValueError( - f"Tokenizer must contain {self._start_of_assistant_message!r} token" - ) - self._start_of_assistant_message_token_id = self.vocab[ - self._start_of_assistant_message - ] - - def is_reasoning_end(self, input_ids: Sequence[int]) -> bool: - # IdentityReasoningParser always returns True: no reasoning to parse. - if isinstance(self._parser, IdentityReasoningParser): - return True - - assert isinstance(self._parser, DeepSeekR1ReasoningParser) - for tok_id in reversed(input_ids): - # : reasoning is not yet ended. - if tok_id == self._parser.start_token_id: - return False - # : reasoning has ended. - if tok_id == self._parser.end_token_id: - return True - # : reached the start of the current assistant turn - # without seeing either marker. Anything further back belongs to - # the prior conversation and should be ignored. - if tok_id == self._start_of_assistant_message_token_id: - return False - return False - - -__all__ = ["PoolsideV1ReasoningParser"] diff --git a/vllm/reasoning/response_template_reasoning_parser.py b/vllm/reasoning/response_template_reasoning_parser.py deleted file mode 100644 index 757dd73417..0000000000 --- a/vllm/reasoning/response_template_reasoning_parser.py +++ /dev/null @@ -1,6 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from vllm.parser.response_template import ResponseTemplateReasoningParser - -__all__ = ["ResponseTemplateReasoningParser"] diff --git a/vllm/reasoning/seed_oss_engine_reasoning_parser.py b/vllm/reasoning/seed_oss_engine_reasoning_parser.py deleted file mode 100644 index e651d411f4..0000000000 --- a/vllm/reasoning/seed_oss_engine_reasoning_parser.py +++ /dev/null @@ -1,6 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from vllm.parser.engine.registered_adapters import SeedOssParserReasoningAdapter - -__all__ = ["SeedOssParserReasoningAdapter"] diff --git a/vllm/reasoning/step3_reasoning_parser.py b/vllm/reasoning/step3_reasoning_parser.py deleted file mode 100644 index e0ada498c2..0000000000 --- a/vllm/reasoning/step3_reasoning_parser.py +++ /dev/null @@ -1,128 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from collections.abc import Iterable, Sequence -from itertools import islice -from typing import TYPE_CHECKING - -import regex as re -from transformers import PreTrainedTokenizerBase - -from vllm.entrypoints.generate.base.protocol import DeltaMessage -from vllm.reasoning import ReasoningParser - -if TYPE_CHECKING: - from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest - from vllm.entrypoints.openai.responses.protocol import ResponsesRequest - - -class Step3ReasoningParser(ReasoningParser): - """Reasoning parser for Step3 model. - - The Step3 model uses token to denote the end of reasoning - text. This parser extracts all content before as reasoning content. - """ - - def __init__(self, tokenizer: PreTrainedTokenizerBase, *args, **kwargs): - super().__init__(tokenizer, *args, **kwargs) - self.think_start_token = "" - self.think_end_token = "" - - self.reasoning_regex = re.compile(rf"(.*?){self.think_end_token}", re.DOTALL) - - if not self.model_tokenizer: - raise ValueError( - "The model tokenizer must be passed to the ReasoningParser " - "constructor during construction." - ) - - think_end_token_id = self.vocab.get(self.think_end_token) - if think_end_token_id is None: - raise RuntimeError( - "Step3 reasoning parser could not locate think end " - "token in the tokenizer!" - ) - self.think_end_token_id: int = think_end_token_id - - @property - def reasoning_start_str(self) -> str: - return self.think_start_token - - @property - def reasoning_end_str(self) -> str: - return self.think_end_token - - def extract_reasoning_streaming( - self, - previous_text: str, - current_text: str, - delta_text: str, - previous_token_ids: Sequence[int], - current_token_ids: Sequence[int], - delta_token_ids: Sequence[int], - ) -> DeltaMessage | None: - """Extract reasoning content from a delta message. - Handles streaming output where previous + delta = current. - Uses token IDs for faster processing. - For text "abcxyz": - - 'abc' goes to reasoning - - 'xyz' goes to content - """ - # Skip single special token - if len(delta_token_ids) == 1 and delta_token_ids[0] == self.think_end_token_id: - return None - - if self.think_end_token_id in delta_token_ids: - # in delta, extract reasoning content and remaining content - end_index = delta_text.find(self.think_end_token) - reasoning = delta_text[:end_index] - content = delta_text[end_index + len(self.think_end_token) :] - return DeltaMessage( - reasoning=reasoning, - content=content if content else None, - ) - elif self.think_end_token_id in previous_token_ids: - # already seen in previous text, everything is content - return DeltaMessage(content=delta_text) - else: - # No seen yet, everything is reasoning - return DeltaMessage(reasoning=delta_text) - - def extract_reasoning( - self, model_output: str, request: "ChatCompletionRequest | ResponsesRequest" - ) -> tuple[str | None, str | None]: - # Check if the model output contains the token - if self.think_end_token not in model_output: - # If no token, everything is reasoning content - return model_output, None - else: - # Find the first occurrence of - end_index = model_output.find(self.think_end_token) - reasoning = model_output[:end_index] - - # Content after token - content = model_output[end_index + len(self.think_end_token) :] or None - - return reasoning, content - - def is_reasoning_end(self, input_ids: Sequence[int]) -> bool: - return self.think_end_token_id in input_ids - - def is_reasoning_end_streaming( - self, input_ids: Sequence[int], delta_ids: Iterable[int] - ) -> bool: - end_token_id = self.think_end_token_id - return end_token_id in delta_ids - - def extract_content_ids(self, input_ids: list[int]) -> list[int]: - if self.think_end_token_id not in islice( - input_ids, 0, max(0, len(input_ids) - 1) - ): - return [] - else: - return input_ids[input_ids.index(self.think_end_token_id) + 1 :] - - def count_reasoning_tokens(self, token_ids: Sequence[int]) -> int: - if self.think_end_token_id not in token_ids: - return len(token_ids) - return list(token_ids).index(self.think_end_token_id) diff --git a/vllm/reasoning/step3p5_engine_reasoning_parser.py b/vllm/reasoning/step3p5_engine_reasoning_parser.py deleted file mode 100644 index 25e9546a1b..0000000000 --- a/vllm/reasoning/step3p5_engine_reasoning_parser.py +++ /dev/null @@ -1,6 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from vllm.parser.engine.registered_adapters import Step3p5ParserReasoningAdapter - -__all__ = ["Step3p5ParserReasoningAdapter"] diff --git a/vllm/third_party/flashmla/__init__.py b/vllm/third_party/flashmla/__init__.py deleted file mode 100644 index f3a161ee70..0000000000 --- a/vllm/third_party/flashmla/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# Sources copied from FlashMLA diff --git a/vllm/tokenizers/deepseek_v32.py b/vllm/tokenizers/deepseek_v32.py deleted file mode 100644 index f933e74cf5..0000000000 --- a/vllm/tokenizers/deepseek_v32.py +++ /dev/null @@ -1,81 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -import copy -from typing import Any - -from transformers import TokenizersBackend - -from vllm.renderers.chat_utils import ChatCompletionMessageParam - -from .deepseek_v32_encoding import encode_messages -from .hf import HfTokenizer, get_cached_tokenizer -from .protocol import TokenizerLike - - -def get_deepseek_v32_tokenizer(tokenizer: HfTokenizer) -> HfTokenizer: - """Wraps a tokenizer to use the custom DeepSeek V3.2 chat template encoding.""" - dsv32_tokenizer = copy.copy(tokenizer) - - added_vocab = tokenizer.get_added_vocab() - - class _DeepseekV32Tokenizer(tokenizer.__class__): # type: ignore - def apply_chat_template( - self, - messages: list["ChatCompletionMessageParam"], - tools: list[dict[str, Any]] | None = None, - **kwargs, - ) -> str | list[int]: - thinking = kwargs.get("thinking", False) - enable_thinking = kwargs.get("enable_thinking", False) - thinking = thinking or enable_thinking - thinking_mode = "thinking" - if not thinking: - thinking_mode = "chat" - conversation = kwargs.get("conversation", messages) - messages = conversation.copy() - if tools is not None and len(tools) > 0: - messages.insert(0, {"role": "system"}) - messages[0]["tools"] = tools # type: ignore[typeddict-unknown-key] - - # Historical reasoning content is dropped when a new user message - # is introduced - drop_thinking = messages[-1]["role"] == "user" - - encode_config = dict( - thinking_mode=thinking_mode, drop_thinking=drop_thinking - ) - - prompt_str = encode_messages(messages, **encode_config) # type: ignore - - if kwargs.get("tokenize", True): - tokenizer_kwargs = { - k: kwargs[k] for k in ("truncation", "max_length") if k in kwargs - } - return self.encode( - prompt_str, - add_special_tokens=False, - **tokenizer_kwargs, - ) - - return prompt_str - - def num_special_tokens_to_add(self) -> int: - return len(self.encode("")) - - def get_added_vocab(self) -> dict[str, int]: - return added_vocab.copy() - - def __reduce__(self): - return get_deepseek_v32_tokenizer, (tokenizer,) - - _DeepseekV32Tokenizer.__name__ = f"DSV32{tokenizer.__class__.__name__}" - - dsv32_tokenizer.__class__ = _DeepseekV32Tokenizer - return dsv32_tokenizer - - -class DeepseekV32Tokenizer(TokenizerLike): - @classmethod - def from_pretrained(cls, *args, **kwargs) -> HfTokenizer: - tokenizer = TokenizersBackend.from_pretrained(*args, **kwargs) - return get_cached_tokenizer(get_deepseek_v32_tokenizer(tokenizer)) diff --git a/vllm/tokenizers/deepseek_v32_encoding.py b/vllm/tokenizers/deepseek_v32_encoding.py deleted file mode 100644 index 75e8488806..0000000000 --- a/vllm/tokenizers/deepseek_v32_encoding.py +++ /dev/null @@ -1,312 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - - -# copy from https://huggingface.co/deepseek-ai/DeepSeek-V3.2/blob/main/encoding/encoding_dsv32.py -import copy -import json -from typing import Any - -# flake8: noqa: E501 -TOOLS_SYSTEM_TEMPLATE = """## Tools -You have access to a set of tools you can use to answer the user's question. -You can invoke functions by writing a "<{dsml_token}function_calls>" block like the following as part of your reply to the user: -<{dsml_token}function_calls> -<{dsml_token}invoke name="$FUNCTION_NAME"> -<{dsml_token}parameter name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE -... - -<{dsml_token}invoke name="$FUNCTION_NAME2"> -... - - -String and scalar parameters should be specified as is without any escaping or quotes, while lists and objects should use JSON format. The "string" attribute should be set to "true" for string type parameters and "false" for other types (numbers, booleans, arrays, objects). -If the thinking_mode is enabled, then after function results you should strongly consider outputting a thinking block. Here is an example: -<{dsml_token}function_calls> -... - - -... - -{thinking_start_token}...thinking about results{thinking_end_token} -Here are the functions available in JSONSchema format: - -{tool_schemas} - -""" - -bos_token: str = "<|begin▁of▁sentence|>" -eos_token: str = "<|end▁of▁sentence|>" -thinking_start_token: str = "" -thinking_end_token: str = "" -dsml_token: str = "|DSML|" -system_msg_template: str = "{content}" -user_msg_template: str = "<|User|>{content}<|Assistant|>" -assistant_msg_template: str = "{reasoning}{content}{tool_calls}<|end▁of▁sentence|>" -thinking_template = "{reasoning}" - -response_format_template: str = "## Response Format:\n\nYou MUST strictly adhere to the following schema to reply:\n{schema}" -tool_call_template: str = ( - '<{dsml_token}invoke name="{name}">\n{arguments}\n' -) -tool_calls_template = ( - "<{dsml_token}function_calls>\n{tool_calls}\n" -) - -tool_output_template: str = "\n{content}" - - -def to_json(value: Any) -> str: - try: - return json.dumps(value, ensure_ascii=False) - except Exception: - return json.dumps(value, ensure_ascii=True) - - -def tools_from_openai_format(tools): - return [tool["function"] for tool in tools] - - -def tool_calls_from_openai_format(tool_calls): - return [ - { - "name": tool_call["function"]["name"], - "arguments": tool_call["function"]["arguments"], - } - for tool_call in tool_calls - ] - - -def encode_arguments_to_dsml(tool_call: dict[str, str]) -> str: - p_dsml_template = """<{dsml_token}parameter name="{key}" string="{is_str}">{value}""" - P_dsml_strs = [] - if isinstance(tool_call["arguments"], str): - arguments = json.loads(tool_call["arguments"]) - else: - arguments = tool_call["arguments"] - - for k, v in arguments.items(): - p_dsml_str = p_dsml_template.format( - dsml_token=dsml_token, - key=k, - is_str="true" if isinstance(v, str) else "false", - value=v if isinstance(v, str) else to_json(v), - ) - - P_dsml_strs.append(p_dsml_str) - - return "\n".join(P_dsml_strs) - - -def render_tools(tools: list[dict[str, str | dict[str, Any]]]) -> str: - tools_json = [to_json(t) for t in tools] - - return TOOLS_SYSTEM_TEMPLATE.format( - tool_schemas="\n".join(tools_json), - dsml_token=dsml_token, - thinking_start_token=thinking_start_token, - thinking_end_token=thinking_end_token, - ) - - -def find_last_user_index(messages: list[dict[str, Any]]) -> int: - last_user_index = -1 - for idx in range(len(messages) - 1, -1, -1): - if messages[idx].get("role") in ["user", "developer"]: - last_user_index = idx - break - return last_user_index - - -def render_message( - index: int, - messages: list[dict[str, Any]], - thinking_mode: str, - last_user_idx: int | None = None, -) -> str: - if not (0 <= index < len(messages)): - raise ValueError( - f"Index {index} out of range for messages list of length {len(messages)}" - ) - if thinking_mode not in ["chat", "thinking"]: - raise ValueError(f"Invalid thinking_mode `{thinking_mode}`") - - prompt = "" - msg = messages[index] - last_user_idx = ( - find_last_user_index(messages) if last_user_idx is None else last_user_idx - ) - - role = msg.get("role") - content = msg.get("content") - tools = msg.get("tools") - response_format = msg.get("response_format") - tool_calls = msg.get("tool_calls") - reasoning = msg.get("reasoning") - is_prefix = msg.get("prefix", False) - - if tools: - tools = tools_from_openai_format(tools) - if tool_calls: - tool_calls = tool_calls_from_openai_format(tool_calls) - - if role == "system": - prompt += system_msg_template.format(content=content or "") - if tools: - prompt += "\n\n" + render_tools(tools) - - if response_format: - prompt += "\n\n" + response_format_template.format( - schema=to_json(response_format) - ) - - elif role == "developer": - if not content: - raise ValueError(f"Invalid message for role `{role}`: {msg}") - content_developer = "" - if tools: - content_developer += "\n\n" + render_tools(tools) - - if response_format: - content_developer += "\n\n" + response_format_template.format( - schema=to_json(response_format) - ) - - content_developer += "\n\n# The user's message is: {}".format(content) - - prompt += user_msg_template.format(content=content_developer) - if index == last_user_idx and thinking_mode == "thinking": - prompt += thinking_start_token - else: - prompt += thinking_end_token - - elif role == "user": - prompt += user_msg_template.format(content=content) - - if index == last_user_idx and thinking_mode == "thinking": - prompt += thinking_start_token - else: - prompt += thinking_end_token - - elif role == "tool": - prev_assistant_idx = index - 1 - assistant_msg = messages[prev_assistant_idx] - while prev_assistant_idx >= 0 and assistant_msg.get("role") == "tool": - prev_assistant_idx -= 1 - assistant_msg = messages[prev_assistant_idx] - - if not ( - index == 0 - or prev_assistant_idx >= 0 - and assistant_msg.get("role") == "assistant" - ): - raise ValueError(f"Invalid messages at {index}:\n{assistant_msg}") - - tool_call_order = index - prev_assistant_idx - assistant_tool_calls = assistant_msg.get("tool_calls") - if not (assistant_tool_calls and len(assistant_tool_calls) >= tool_call_order): - raise ValueError("No tool calls but found tool output") - - if tool_call_order == 1: - prompt += "\n\n" - - prompt += tool_output_template.format(content=content) - - if tool_call_order == len(assistant_tool_calls): - prompt += "\n" - - if index >= last_user_idx and thinking_mode == "thinking": - prompt += "\n\n" + thinking_start_token - else: - prompt += "\n\n" + thinking_end_token - - elif role == "assistant": - prev_assistant_idx = index - thinking_part = "" - - tool_calls_content = "" - if tool_calls: - tool_calls = [ - tool_call_template.format( - dsml_token=dsml_token, - name=tool_call.get("name"), - arguments=encode_arguments_to_dsml(tool_call), - ) - for tool_call in tool_calls - ] - tool_calls_content += "\n\n" + tool_calls_template.format( - dsml_token=dsml_token, tool_calls="\n".join(tool_calls) - ) - - summary_content = content or "" - - if thinking_mode == "thinking" and index > last_user_idx: - if not (reasoning or tool_calls): - raise ValueError( - f"ThinkingMode: {thinking_mode}, invalid message without reasoning/tool_calls `{msg}` after last user message" - ) - thinking_part = ( - thinking_template.format(reasoning=reasoning or "") + thinking_end_token - ) - - if not tool_calls and is_prefix: - prompt += summary_content - else: - prompt += assistant_msg_template.format( - reasoning=thinking_part, - content=summary_content, - tool_calls=tool_calls_content, - ) - else: - raise ValueError(f"Invalid role: {role}") - - return prompt - - -def drop_thinking_messages( - messages: list[dict[str, Any]], last_user_idx: int | None = None -) -> list[dict[str, Any]]: - messages_wo_thinking: list[dict[str, Any]] = [] - last_user_idx = ( - find_last_user_index(messages) if last_user_idx is None else last_user_idx - ) - for idx, msg in enumerate(messages): - role = msg.get("role") - if role in ["user", "system", "tool"] or idx >= last_user_idx: - messages_wo_thinking.append(msg) - continue - - elif role == "assistant": - msg_wo_thinking = copy.copy(msg) - msg_wo_thinking.pop("reasoning", None) - messages_wo_thinking.append(msg_wo_thinking) - - return messages_wo_thinking - - -def encode_messages( - messages: list[dict[str, Any]], - thinking_mode: str, - context: list[dict[str, Any]] | None = None, - drop_thinking: bool = True, - add_default_bos_token: bool = True, -) -> str: - context = context if context else [] - full_messages = context + messages - - prompt = bos_token if add_default_bos_token and len(context) == 0 else "" - - if thinking_mode == "thinking" and drop_thinking: - full_messages = drop_thinking_messages(full_messages) - - last_user_idx = find_last_user_index(full_messages) - - for idx in range(len(messages)): - prompt += render_message( - idx + len(context), - full_messages, - thinking_mode=thinking_mode, - last_user_idx=last_user_idx, - ) - - return prompt diff --git a/vllm/tokenizers/deepseek_v4.py b/vllm/tokenizers/deepseek_v4.py deleted file mode 100644 index 471d6fbb4a..0000000000 --- a/vllm/tokenizers/deepseek_v4.py +++ /dev/null @@ -1,103 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -import copy -from typing import Any - -from transformers import TokenizersBackend - -from vllm.renderers.chat_utils import ChatCompletionMessageParam - -from .deepseek_v4_encoding import encode_messages -from .hf import HfTokenizer, get_cached_tokenizer -from .protocol import TokenizerLike - - -def get_deepseek_v4_tokenizer(tokenizer: HfTokenizer) -> HfTokenizer: - """Wraps a tokenizer to use the custom DeepSeek V4 chat template encoding.""" - dsv4_tokenizer = copy.copy(tokenizer) - - added_vocab = tokenizer.get_added_vocab() - - class _DeepseekV4Tokenizer(tokenizer.__class__): # type: ignore - def apply_chat_template( - self, - messages: list["ChatCompletionMessageParam"], - tools: list[dict[str, Any]] | None = None, - **kwargs, - ) -> str | list[int]: - thinking = kwargs.get("thinking") - enable_thinking = kwargs.get("enable_thinking") - thinking_enabled = bool(thinking) or bool(enable_thinking) - if "thinking" not in kwargs and "enable_thinking" not in kwargs: - thinking_enabled = True - thinking_mode = "thinking" if thinking_enabled else "chat" - - conversation = kwargs.get("conversation", messages) - messages = conversation.copy() - if tools is not None and len(tools) > 0: - # Match the Rust renderer: request tools attach to the first - # system message; synthesize one only when none exists. - system_idx = next( - (i for i, m in enumerate(messages) if m.get("role") == "system"), - None, - ) - if system_idx is None: - messages.insert(0, {"role": "system"}) - system_idx = 0 - else: - messages[system_idx] = copy.copy(messages[system_idx]) - messages[system_idx]["tools"] = tools # type: ignore[typeddict-unknown-key] - - reasoning_effort = kwargs.get("reasoning_effort") - if not isinstance(reasoning_effort, str): - reasoning_effort = "high" if thinking_enabled else None - elif reasoning_effort == "none": - thinking_mode = "chat" - reasoning_effort = None - elif reasoning_effort == "max": - reasoning_effort = "max" - elif reasoning_effort in ("low", "minimal", "medium"): - reasoning_effort = "low" - else: - reasoning_effort = "high" - - encode_config = dict( - thinking_mode=thinking_mode, - drop_thinking=kwargs.get("drop_thinking", True), - reasoning_effort=reasoning_effort, - ) - - prompt_str = encode_messages(messages, **encode_config) # type: ignore - - if kwargs.get("tokenize", True): - tokenizer_kwargs = { - k: kwargs[k] for k in ("truncation", "max_length") if k in kwargs - } - return self.encode( - prompt_str, - add_special_tokens=False, - **tokenizer_kwargs, - ) - - return prompt_str - - def num_special_tokens_to_add(self) -> int: - return len(self.encode("")) - - def get_added_vocab(self) -> dict[str, int]: - return added_vocab.copy() - - def __reduce__(self): - return get_deepseek_v4_tokenizer, (tokenizer,) - - _DeepseekV4Tokenizer.__name__ = f"DSV4{tokenizer.__class__.__name__}" - - dsv4_tokenizer.__class__ = _DeepseekV4Tokenizer - return dsv4_tokenizer - - -class DeepseekV4Tokenizer(TokenizerLike): - @classmethod - def from_pretrained(cls, *args, **kwargs) -> HfTokenizer: - tokenizer = TokenizersBackend.from_pretrained(*args, **kwargs) - return get_cached_tokenizer(get_deepseek_v4_tokenizer(tokenizer)) diff --git a/vllm/tokenizers/deepseek_v41_encoding.py b/vllm/tokenizers/deepseek_v41_encoding.py deleted file mode 100644 index 6851d42f91..0000000000 --- a/vllm/tokenizers/deepseek_v41_encoding.py +++ /dev/null @@ -1,577 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -# ruff: noqa - -"""Text encoding adapted from the DeepSeek V4.1 reference encoder. - -Message normalization and serving controls live in deepseek_v41.py. -""" - -import copy -import json -from typing import Any, Dict, List, Optional, Union - -bos_token: str = "<|begin▁of▁sentence|>" -eos_token: str = "<|end▁of▁sentence|>" -thinking_start_token: str = "" -thinking_end_token: str = "" -dsml_token: str = "|DSML|" - -USER_SP_TOKEN = "<|User|>" -ASSISTANT_SP_TOKEN = "<|Assistant|>" -LATEST_REMINDER_SP_TOKEN = "<|latest_reminder|>" -IMAGE_PLACEHOLDER = "<|deepseek_image|>" - -DS_TASK_SP_TOKENS = { - "action": "<|action|>", - "query": "<|query|>", - "authority": "<|authority|>", - "domain": "<|domain|>", - "title": "<|title|>", - "read_url": "<|read_url|>", -} -VALID_TASKS = set(DS_TASK_SP_TOKENS.keys()) - -system_msg_template: str = "{content}" -user_msg_template: str = "{content}" -latest_reminder_msg_template: str = "{content}" -assistant_msg_template: str = "{reasoning}{content}{tool_calls}" + eos_token -assistant_msg_wo_eos_template: str = "{reasoning}{content}{tool_calls}" -thinking_template: str = "{reasoning_content}" - -response_format_template: str = ( - "## Response Format:\n\n" - "You MUST strictly adhere to the following schema to reply:\n{schema}" -) -tool_output_template: str = "{content}" - - -def to_json(value: Any) -> str: - """Serialize a value to JSON string.""" - try: - return json.dumps(value, ensure_ascii=False) - except: - return json.dumps(value, ensure_ascii=True) - - -def tools_from_openai_format(tools): - """Extract function definitions from OpenAI-format tool list.""" - return [tool["function"] for tool in tools] - - -def tool_calls_from_openai_format(tool_calls): - """Convert OpenAI-format tool calls to internal format.""" - return [ - { - "name": tool_call["function"]["name"], - "arguments": tool_call["function"]["arguments"], - } - for tool_call in tool_calls - ] - - -def merge_tool_messages(messages: List[Dict[str, Any]]) -> List[Dict[str, Any]]: - """ - Merge tool messages into the preceding user message using content_blocks format. - - DeepSeek-V4.1 does not have a standalone "tool" role; instead, tool results - are encoded as blocks within user messages. - """ - merged: List[Dict[str, Any]] = [] - - for msg in messages: - msg = copy.deepcopy(msg) - role = msg.get("role") - - if role == "tool": - # Convert tool message to a user message with tool_result block - tool_block = { - "type": "tool_result", - "tool_use_id": msg.get("tool_call_id", ""), - "content": msg.get("content", ""), - } - # Merge into previous message if it's already a user (merged tool) - if ( - merged - and merged[-1].get("role") == "user" - and "content_blocks" in merged[-1] - ): - merged[-1]["content_blocks"].append(tool_block) - else: - merged.append( - { - "role": "user", - "content_blocks": [tool_block], - } - ) - elif role == "user": - content_blocks = msg.get("content_blocks") - if content_blocks is None: - content_blocks = [{"type": "text", "text": msg.get("content", "")}] - if ( - merged - and merged[-1].get("role") == "user" - and "content_blocks" in merged[-1] - and merged[-1].get("task") is None - ): - merged[-1]["content_blocks"].extend(content_blocks) - else: - # Preserve structured content and all message-level metadata. - new_msg = msg - new_msg["content_blocks"] = content_blocks - merged.append(new_msg) - else: - merged.append(msg) - - return merged - - -def sort_tool_results_by_call_order( - messages: List[Dict[str, Any]], -) -> List[Dict[str, Any]]: - """ - Sort tool_result blocks within user messages by the order of tool_calls - in the preceding assistant message. - """ - last_tool_call_order: Dict[str, int] = {} - - for msg in messages: - role = msg.get("role") - if role == "assistant" and msg.get("tool_calls"): - last_tool_call_order = {} - for idx, tc in enumerate(msg["tool_calls"]): - tc_id = tc.get("id") or tc.get("function", {}).get("id", "") - if tc_id: - last_tool_call_order[tc_id] = idx - - elif role == "user" and msg.get("content_blocks"): - tool_blocks = [ - b for b in msg["content_blocks"] if b.get("type") == "tool_result" - ] - if len(tool_blocks) > 1 and last_tool_call_order: - sorted_blocks = sorted( - tool_blocks, - key=lambda b: last_tool_call_order.get(b.get("tool_use_id", ""), 0), - ) - sorted_idx = 0 - new_blocks = [] - for block in msg["content_blocks"]: - if block.get("type") == "tool_result": - new_blocks.append(sorted_blocks[sorted_idx]) - sorted_idx += 1 - else: - new_blocks.append(block) - msg["content_blocks"] = new_blocks - - return messages - - -SYSTEM_SP_TOKEN = "<|System|>" - -tool_calls_block_name: str = " calls" -tool_call_tag_name: str = " invoke" -tool_parameter_tag_name: str = " parameter" -tool_call_template: str = '<{dsml_token}{tool_call_tag_name} name="{name}">\n{arguments}\n' -tool_calls_template = ( - "<{dsml_token}{tc_block_name}>\n{tool_calls}\n" -) - -REASONING_EFFORT_TEMPLATE = ( - "Reasoning Effort: {budget} " - "(range 1-100, the higher the value, the more thorough the reasoning)\n\n" -) -REASONING_EFFORT_MAPPINGS: Dict[str, int] = { - "low": 50, - "high": 75, - "xhigh": 75, - "max": 100, -} - -DEFAULT_REASONING_EFFORT = "high" - - -def render_reasoning_effort( - index: int, - thinking_mode: str, - effort: Union[str, int, None], -) -> str: - """Render the V4.1 numeric reasoning effort prefix (thinking mode, index 0 only).""" - if effort is None: - effort = DEFAULT_REASONING_EFFORT - assert ( - type(effort) is int and 1 <= effort <= 100 - ) or effort in REASONING_EFFORT_MAPPINGS, ( - "Invalid reasoning effort for deepseek_v41: " - f"{effort}, should be int within [1,100] or {list(REASONING_EFFORT_MAPPINGS)}" - ) - if type(effort) is str: - effort = REASONING_EFFORT_MAPPINGS[effort] - if index == 0 and thinking_mode == "thinking": - return REASONING_EFFORT_TEMPLATE.format(budget=effort) - return "" - - -TOOLS_TEMPLATE = """## Tools - -You have access to a set of tools to help answer the user's question. You can invoke tools by writing a "<{dsml_token}{tc_block_name}>" block like the following: - -<{dsml_token}{tc_block_name}> -<{dsml_token}{tool_call_tag_name} name="$TOOL_NAME"> -<{dsml_token}{tool_parameter_tag_name} name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE -... - -<{dsml_token}{tool_call_tag_name} name="$TOOL_NAME2"> -... - - - -String parameters should be specified as is and set `string="true"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string="false"`. - -If thinking_mode is enabled (triggered by {thinking_start_token}), you MUST output your complete reasoning inside {thinking_start_token}...{thinking_end_token} BEFORE any tool calls or final response. - -Otherwise, output directly after {thinking_end_token} with tool calls or final response. - -### Available Tool Schemas - -{tool_schemas} - -You MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls. -""" - - -def render_tools(tools: List[Dict[str, Union[str, Dict[str, Any]]]]) -> str: - """Render tool schemas into the V4.1 system prompt format.""" - tools_json = [to_json(t) for t in tools] - - return TOOLS_TEMPLATE.format( - tool_schemas="\n".join(tools_json), - dsml_token=dsml_token, - tc_block_name=tool_calls_block_name, - tool_call_tag_name=tool_call_tag_name, - tool_parameter_tag_name=tool_parameter_tag_name, - thinking_start_token=thinking_start_token, - thinking_end_token=thinking_end_token, - ) - - -def encode_arguments_to_dsml(tool_call: Dict[str, Any]) -> str: - """Encode tool call arguments into V4.1 DSML parameter format.""" - p_dsml_template = ( - '<{dsml_token}{tool_parameter_tag_name} name="{key}" string="{is_str}">' - "{value}" - ) - P_dsml_strs = [] - - arguments = tool_call["arguments"] - if not isinstance(arguments, dict): - # Tolerate JSON strings, including double-encoded ones. - for _ in range(2): - if isinstance(arguments, str): - try: - arguments = json.loads(arguments) - except Exception: - break - else: - break - if not isinstance(arguments, dict): - arguments = {"arguments": tool_call["arguments"]} - - for k, v in arguments.items(): - P_dsml_strs.append( - p_dsml_template.format( - dsml_token=dsml_token, - tool_parameter_tag_name=tool_parameter_tag_name, - key=k, - is_str="true" if isinstance(v, str) else "false", - value=v if isinstance(v, str) else to_json(v), - ) - ) - - return "\n".join(P_dsml_strs) - - -def find_last_user_index(messages: List[Dict[str, Any]]) -> int: - """ - Find the index of the last user/developer message. - - V4.1 supports mid-conversation system messages, which count as user - messages for the purposes of the assistant generation header. - """ - last_user_index = -1 - for idx in range(len(messages) - 1, -1, -1): - role = messages[idx].get("role") - if role in ["user", "developer"] or (role == "system" and idx > 0): - last_user_index = idx - break - return last_user_index - - -def render_message( - index: int, - messages: List[Dict[str, Any]], - thinking_mode: str, - drop_thinking: bool = True, - reasoning_effort: Union[str, int, None] = None, -) -> str: - """ - Render a single message at the given index into its V4.1 encoded string form. - """ - assert 0 <= index < len(messages) - assert thinking_mode in ["chat", "thinking"], ( - f"Invalid thinking_mode `{thinking_mode}`" - ) - - msg = messages[index] - last_user_idx = find_last_user_index(messages) - - role = msg.get("role") - content = msg.get("content") - tools = msg.get("tools") - response_format = msg.get("response_format") - tool_calls = msg.get("tool_calls") - reasoning_content = msg.get("reasoning_content") - wo_eos = msg.get("wo_eos", False) - - if tools: - tools = tools_from_openai_format(tools) - if tool_calls: - tool_calls = tool_calls_from_openai_format(tool_calls) - - # Reasoning effort prefix (thinking mode, index 0 only) - reasoning_effort_prompt = render_reasoning_effort( - index, thinking_mode, reasoning_effort - ) - # System token leads the conversation when there is a reasoning effort prompt - # or the first message is a system message. - prompt = ( - SYSTEM_SP_TOKEN - if index == 0 and (reasoning_effort_prompt or role == "system") - else "" - ) - prompt += reasoning_effort_prompt - - if role == "system": - if index > 0: - # Mid-conversation system message - prompt += SYSTEM_SP_TOKEN - prompt += system_msg_template.format(content=content or "") - if tools: - prompt += "\n\n" + render_tools(tools) - if response_format: - prompt += "\n\n" + response_format_template.format( - schema=to_json(response_format) - ) - - elif role == "developer": - assert content, f"Invalid message for role `{role}`: {msg}" - - content_developer = USER_SP_TOKEN - content_developer += content - - if tools: - content_developer += "\n\n" + render_tools(tools) - if response_format: - content_developer += "\n\n" + response_format_template.format( - schema=to_json(response_format) - ) - - prompt += user_msg_template.format(content=content_developer) - - elif role == "user": - prompt += USER_SP_TOKEN - - # Handle content blocks (tool results mixed with text) - content_blocks = msg.get("content_blocks") - if content_blocks: - parts = [] - for block in content_blocks: - block_type = block.get("type") - if block_type == "text": - parts.append(block.get("text", "")) - elif block_type == "tool_result": - tool_content = block.get("content", "") - if isinstance(tool_content, list): - text_parts = [] - for b in tool_content: - if b.get("type") == "text": - text_parts.append(b.get("text", "")) - else: - text_parts.append(f"[Unsupported {b.get('type')}]") - tool_content = "\n\n".join(text_parts) - parts.append(tool_output_template.format(content=tool_content)) - else: - parts.append(f"[Unsupported {block_type}]") - prompt += "\n\n".join(parts) - else: - prompt += content or "" - - elif role == "latest_reminder": - prompt += LATEST_REMINDER_SP_TOKEN + latest_reminder_msg_template.format( - content=content - ) - - elif role == "tool": - raise NotImplementedError( - "deepseek_v41 merges tool messages into user; please preprocess with merge_tool_messages()" - ) - - elif role == "assistant": - thinking_part = "" - tc_content = "" - - if tool_calls: - tc_list = [ - tool_call_template.format( - dsml_token=dsml_token, - tool_call_tag_name=tool_call_tag_name, - name=tc.get("name"), - arguments=encode_arguments_to_dsml(tc), - ) - for tc in tool_calls - ] - tc_content += "\n\n" + tool_calls_template.format( - dsml_token=dsml_token, - tool_calls="\n".join(tc_list), - tc_block_name=tool_calls_block_name, - ) - - summary_content = content or "" - rc = reasoning_content or "" - - # Check if previous message has a task - if so, this is a task output (no thinking) - prev_has_task = index - 1 >= 0 and messages[index - 1].get("task") is not None - - if thinking_mode == "thinking" and not prev_has_task: - if not drop_thinking or index > last_user_idx: - thinking_part = ( - thinking_template.format(reasoning_content=rc) + thinking_end_token - ) - else: - thinking_part = "" - - if wo_eos: - prompt += assistant_msg_wo_eos_template.format( - reasoning=thinking_part, - content=summary_content, - tool_calls=tc_content, - ) - else: - prompt += assistant_msg_template.format( - reasoning=thinking_part, - content=summary_content, - tool_calls=tc_content, - ) - else: - raise NotImplementedError(f"Unknown role: {role}") - - # Append transition tokens based on what follows - if index + 1 < len(messages) and messages[index + 1].get("role") not in [ - "assistant", - "latest_reminder", - ]: - return prompt - - task = messages[index].get("task") - if task is not None: - # Task special token for internal classification tasks - assert task in VALID_TASKS, ( - f"Invalid task: '{task}'. Valid tasks are: {list(VALID_TASKS)}" - ) - task_sp_token = DS_TASK_SP_TOKENS[task] - - if task != "action": - # Non-action tasks: append task sp token directly after the message - prompt += task_sp_token - else: - # Action task: append Assistant + thinking token + action sp token - prompt += ASSISTANT_SP_TOKEN - prompt += ( - thinking_end_token - if thinking_mode != "thinking" - else thinking_start_token - ) - prompt += task_sp_token - - elif messages[index].get("role") in ["user", "developer"] or ( - messages[index].get("role") == "system" and index > 0 - ): - # Normal generation: append Assistant + thinking token - # (mid-conversation system messages also trigger the assistant header) - prompt += ASSISTANT_SP_TOKEN - if not drop_thinking and thinking_mode == "thinking": - prompt += thinking_start_token - elif drop_thinking and thinking_mode == "thinking" and index >= last_user_idx: - prompt += thinking_start_token - else: - prompt += thinking_end_token - - return prompt - - -def _drop_thinking_messages(messages: List[Dict[str, Any]]) -> List[Dict[str, Any]]: - """ - Drop reasoning_content and non-essential messages before the last user message. - Same as V4, but uses the V4.1 last-user definition (mid systems count). - """ - last_user_idx = find_last_user_index(messages) - result = [] - keep_roles = {"user", "system", "tool", "latest_reminder", "direct_search_results"} - - for idx, msg in enumerate(messages): - role = msg.get("role") - if role in keep_roles or idx >= last_user_idx: - result.append(msg) - elif role == "assistant": - msg = copy.copy(msg) - msg.pop("reasoning_content", None) - result.append(msg) - # developer and other roles before last_user_idx are dropped - - return result - - -def encode_messages( - messages: List[Dict[str, Any]], - thinking_mode: str, - context: Optional[List[Dict[str, Any]]] = None, - drop_thinking: bool = True, - add_default_bos_token: bool = True, - reasoning_effort: Union[str, int, None] = None, -) -> str: - """Encode preprocessed (text-only) messages into the V4.1 prompt format.""" - context = context if context else [] - - # Preprocess: merge tool messages and sort tool results - messages = merge_tool_messages(messages) - messages = sort_tool_results_by_call_order(context + messages)[len(context) :] - if context: - context = merge_tool_messages(context) - context = sort_tool_results_by_call_order(context) - - full_messages = context + messages - - prompt = bos_token if add_default_bos_token and len(context) == 0 else "" - - # Resolve drop_thinking: if any message has tools defined, don't drop thinking - effective_drop_thinking = drop_thinking - if any(m.get("tools") for m in full_messages): - effective_drop_thinking = False - - if thinking_mode == "thinking" and effective_drop_thinking: - full_messages = _drop_thinking_messages(full_messages) - num_to_render = len(full_messages) - len(_drop_thinking_messages(context)) - context_len = len(full_messages) - num_to_render - else: - num_to_render = len(messages) - context_len = len(context) - - for idx in range(num_to_render): - prompt += render_message( - idx + context_len, - full_messages, - thinking_mode=thinking_mode, - drop_thinking=effective_drop_thinking, - reasoning_effort=reasoning_effort, - ) - - return prompt diff --git a/vllm/tokenizers/deepseek_v4_encoding.py b/vllm/tokenizers/deepseek_v4_encoding.py deleted file mode 100644 index 37dc559366..0000000000 --- a/vllm/tokenizers/deepseek_v4_encoding.py +++ /dev/null @@ -1,648 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -# ruff: noqa -# fmt: off - -""" -DeepSeek-V4 Encoding - -A self-contained implementation for encoding DeepSeek-V4 chat messages with tool -calling, thinking mode, and quick instruction task support. -""" - -from typing import Any, Dict, List, Union, Optional -import copy -import json - -# ============================================================ -# Special Tokens -# ============================================================ - -bos_token: str = "<|begin▁of▁sentence|>" -eos_token: str = "<|end▁of▁sentence|>" -thinking_start_token: str = "" -thinking_end_token: str = "" -dsml_token: str = "|DSML|" - -USER_SP_TOKEN = "<|User|>" -ASSISTANT_SP_TOKEN = "<|Assistant|>" -LATEST_REMINDER_SP_TOKEN = "<|latest_reminder|>" - -# Placeholder text inlined at each image's position; the multimodal processor -# later expands it into the sentinel token block. -IMAGE_PLACEHOLDER = "<|deepseek_image|>" - -# Task special tokens for internal classification tasks -DS_TASK_SP_TOKENS = { - "action": "<|action|>", - "query": "<|query|>", - "authority": "<|authority|>", - "domain": "<|domain|>", - "title": "<|title|>", - "read_url": "<|read_url|>", -} -VALID_TASKS = set(DS_TASK_SP_TOKENS.keys()) - -# ============================================================ -# Templates -# ============================================================ - -system_msg_template: str = "{content}" -user_msg_template: str = "{content}" -latest_reminder_msg_template: str = "{content}" -assistant_msg_template: str = "{reasoning}{content}{tool_calls}" + eos_token -assistant_msg_wo_eos_template: str = "{reasoning}{content}{tool_calls}" -thinking_template: str = "{reasoning}" - -response_format_template: str = ( - "## Response Format:\n\nYou MUST strictly adhere to the following schema to reply:\n{schema}" -) -tool_call_template: str = ( - "<{dsml_token}invoke name=\"{name}\">\n{arguments}\n" -) -tool_calls_template = ( - "<{dsml_token}{tc_block_name}>\n{tool_calls}\n" -) -tool_calls_block_name: str = "tool_calls" - -tool_output_template: str = ( - "{content}" -) - -REASONING_EFFORT_PROMPTS: Dict[str, str] = { - "low": "", - "high": ( - "Reasoning Effort: Absolute maximum with no shortcuts permitted.\n" - "You MUST be very thorough in your thinking and comprehensively decompose the problem to resolve the root cause, rigorously stress-testing your logic against all potential paths, edge cases, and adversarial scenarios.\n" - "Explicitly write out your entire deliberation process, documenting every intermediate step, considered alternative, and rejected hypothesis to ensure absolutely no assumption is left unchecked.\n\n" - ), - "max": ( - "Reasoning Effort: Beyond maximum — exhaustive, relentless, and uncompromising.\n" - "You MUST reason with the utmost depth and rigor, leaving absolutely nothing to chance: exhaustively decompose the problem into its most fundamental components, trace every causal chain to its root, and resolve the underlying cause rather than any surface symptom.\n" - "Do not stop reasoning until you have independently verified the solution from multiple angles and are certain that no assumption remains unchecked and no error remains undiscovered.\n\n" - ), -} -DEFAULT_REASONING_EFFORT = "low" - -TOOLS_TEMPLATE = """## Tools - -You have access to a set of tools to help answer the user's question. You can invoke tools by writing a "<{dsml_token}tool_calls>" block like the following: - -<{dsml_token}tool_calls> -<{dsml_token}invoke name="$TOOL_NAME"> -<{dsml_token}parameter name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE -... - -<{dsml_token}invoke name="$TOOL_NAME2"> -... - - - -String parameters should be specified as is and set `string="true"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string="false"`. - -If thinking_mode is enabled (triggered by {thinking_start_token}), you MUST output your complete reasoning inside {thinking_start_token}...{thinking_end_token} BEFORE any tool calls or final response. - -Otherwise, output directly after {thinking_end_token} with tool calls or final response. - -### Available Tool Schemas - -{tool_schemas} - -You MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls. -""" - -# ============================================================ -# Utility Functions -# ============================================================ - -def to_json(value: Any) -> str: - """Serialize a value to JSON string.""" - try: - return json.dumps(value, ensure_ascii=False) - except Exception: - return json.dumps(value, ensure_ascii=True) - - -def tools_from_openai_format(tools): - """Extract function definitions from OpenAI-format tool list.""" - return [tool["function"] for tool in tools] - - -def tool_calls_from_openai_format(tool_calls): - """Convert OpenAI-format tool calls to internal format.""" - return [ - { - "name": tool_call["function"]["name"], - "arguments": tool_call["function"]["arguments"], - } - for tool_call in tool_calls - ] - - -def encode_arguments_to_dsml(tool_call: Dict[str, Any]) -> str: - """ - Encode tool call arguments into DSML parameter format. - - Args: - tool_call: Dict with "name" and "arguments" keys. - - Returns: - DSML-formatted parameter string. - """ - p_dsml_template = '<{dsml_token}parameter name="{key}" string="{is_str}">{value}' - P_dsml_strs = [] - - if isinstance(tool_call["arguments"], str): - arguments = json.loads(tool_call["arguments"]) - else: - arguments = tool_call["arguments"] - - for k, v in arguments.items(): - p_dsml_str = p_dsml_template.format( - dsml_token=dsml_token, - key=k, - is_str="true" if isinstance(v, str) else "false", - value=v if isinstance(v, str) else to_json(v), - ) - P_dsml_strs.append(p_dsml_str) - - return "\n".join(P_dsml_strs) - - -def render_tools(tools: List[Dict[str, Union[str, Dict[str, Any]]]]) -> str: - """ - Render tool schemas into the system prompt format. - - Args: - tools: List of tool schema dicts (each with name, description, parameters). - - Returns: - Formatted tools section string. - """ - tools_json = [to_json(t) for t in tools] - - return TOOLS_TEMPLATE.format( - tool_schemas="\n".join(tools_json), - dsml_token=dsml_token, - thinking_start_token=thinking_start_token, - thinking_end_token=thinking_end_token, - ) - - -def find_last_user_index(messages: List[Dict[str, Any]]) -> int: - """Find the index of the last user/developer message.""" - last_user_index = -1 - for idx in range(len(messages) - 1, -1, -1): - if messages[idx].get("role") in ["user", "developer"]: - last_user_index = idx - break - return last_user_index - - -# ============================================================ -# Message Rendering -# ============================================================ - -def flatten_content_blocks(content: Any) -> Any: - """Flatten OpenAI-style content blocks to plain text. - - Image blocks are inlined as IMAGE_PLACEHOLDER at their position. The image - data itself travels out of band (multi_modal_data) and is matched to - placeholders by order. Plain-string content passes through unchanged. - """ - if not isinstance(content, list): - return content - parts: List[str] = [] - for block in content: - if not isinstance(block, dict): - parts.append(str(block)) - elif block.get("type") in ("image", "image_url"): - parts.append(IMAGE_PLACEHOLDER) - elif block.get("type") == "text": - parts.append(block.get("text", "")) - else: - raise ValueError( - f"Unsupported content block type: {block.get('type')!r}" - ) - return "\n\n".join(parts) - - -def render_message( - index: int, - messages: List[Dict[str, Any]], - thinking_mode: str, - drop_thinking: bool = True, - reasoning_effort: Optional[str] = None, - last_user_idx: Optional[int] = None, -) -> str: - """ - Render a single message at the given index into its encoded string form. - - This is the core function that converts each message in the conversation - into the DeepSeek-V4 format. - - Args: - index: Index of the message to render. - messages: Full list of messages in the conversation. - thinking_mode: Either "chat" or "thinking". - drop_thinking: Whether to drop reasoning content from earlier turns. - reasoning_effort: Reasoning effort level, one of "low", "high", "max". - None is treated as "low". - last_user_idx: Cached index of the last user/developer message. - - Returns: - Encoded string for this message. - """ - if not (0 <= index < len(messages)): - raise ValueError( - f"Index {index} out of range for messages list of length {len(messages)}" - ) - if thinking_mode not in ["chat", "thinking"]: - raise ValueError(f"Invalid thinking_mode `{thinking_mode}`") - - prompt = "" - msg = messages[index] - last_user_idx = ( - find_last_user_index(messages) if last_user_idx is None else last_user_idx - ) - - role = msg.get("role") - content = flatten_content_blocks(msg.get("content")) - tools = msg.get("tools") - response_format = msg.get("response_format") - tool_calls = msg.get("tool_calls") - reasoning = msg.get("reasoning") - wo_eos = msg.get("wo_eos", False) - - if tools: - tools = tools_from_openai_format(tools) - if tool_calls: - tool_calls = tool_calls_from_openai_format(tool_calls) - - reasoning_effort = reasoning_effort or DEFAULT_REASONING_EFFORT - if reasoning_effort not in REASONING_EFFORT_PROMPTS: - raise ValueError( - f"Invalid reasoning effort: {reasoning_effort}, expected one of " - f"{list(REASONING_EFFORT_PROMPTS)}" - ) - if index == 0 and thinking_mode == "thinking": - prompt += REASONING_EFFORT_PROMPTS[reasoning_effort] - - if role == "system": - prompt += system_msg_template.format(content=content or "") - if tools: - prompt += "\n\n" + render_tools(tools) - if response_format: - prompt += "\n\n" + response_format_template.format(schema=to_json(response_format)) - - elif role == "developer": - if not content: - raise ValueError(f"Invalid message for role `{role}`: {msg}") - - content_developer = USER_SP_TOKEN - content_developer += content - - if tools: - content_developer += "\n\n" + render_tools(tools) - if response_format: - content_developer += "\n\n" + response_format_template.format(schema=to_json(response_format)) - - prompt += user_msg_template.format(content=content_developer) - - elif role == "user": - prompt += USER_SP_TOKEN - - # Handle content blocks (tool results mixed with text) - content_blocks = msg.get("content_blocks") - if content_blocks: - parts = [] - for block in content_blocks: - block_type = block.get("type") - if block_type == "text": - parts.append(block.get("text", "")) - elif block_type in ("image", "image_url"): - parts.append(IMAGE_PLACEHOLDER) - elif block_type == "tool_result": - tool_content = block.get("content", "") - if isinstance(tool_content, list): - text_parts = [] - for b in tool_content: - if b.get("type") == "text": - text_parts.append(b.get("text", "")) - else: - text_parts.append(f"[Unsupported {b.get('type')}]") - tool_content = "\n\n".join(text_parts) - parts.append(tool_output_template.format(content=tool_content)) - else: - parts.append(f"[Unsupported {block_type}]") - prompt += "\n\n".join(parts) - else: - prompt += content or "" - - elif role == "latest_reminder": - prompt += LATEST_REMINDER_SP_TOKEN + latest_reminder_msg_template.format(content=content) - - elif role == "tool": - raise NotImplementedError("deepseek_v4 merges tool messages into user; please preprocess with merge_tool_messages()") - - elif role == "assistant": - thinking_part = "" - tc_content = "" - - if tool_calls: - tc_list = [ - tool_call_template.format( - dsml_token=dsml_token, - name=tc.get("name"), - arguments=encode_arguments_to_dsml(tc) - ) - for tc in tool_calls - ] - tc_content += '\n\n' + tool_calls_template.format( - dsml_token=dsml_token, - tool_calls="\n".join(tc_list), - tc_block_name=tool_calls_block_name, - ) - - summary_content = content or "" - reasoning = reasoning or "" - - # Check if previous message has a task - if so, this is a task output (no thinking) - prev_has_task = index - 1 >= 0 and messages[index - 1].get("task") is not None - - if thinking_mode == "thinking" and not prev_has_task: - if not drop_thinking or index > last_user_idx: - thinking_part = thinking_template.format(reasoning=reasoning) + thinking_end_token - else: - thinking_part = "" - - if wo_eos: - prompt += assistant_msg_wo_eos_template.format( - reasoning=thinking_part, - content=summary_content, - tool_calls=tc_content, - ) - else: - prompt += assistant_msg_template.format( - reasoning=thinking_part, - content=summary_content, - tool_calls=tc_content, - ) - else: - raise ValueError(f"Invalid role: {role}") - - # Append transition tokens based on what follows - if index + 1 < len(messages) and messages[index + 1].get("role") not in ["assistant", "latest_reminder"]: - return prompt - - task = messages[index].get("task") - if task is not None: - # Task special token for internal classification tasks - if task not in VALID_TASKS: - raise ValueError( - f"Invalid task: '{task}'. Valid tasks are: {list(VALID_TASKS)}" - ) - task_sp_token = DS_TASK_SP_TOKENS[task] - - if task != "action": - # Non-action tasks: append task sp token directly after the message - prompt += task_sp_token - else: - # Action task: append Assistant + thinking token + action sp token - prompt += ASSISTANT_SP_TOKEN - prompt += thinking_end_token if thinking_mode != "thinking" else thinking_start_token - prompt += task_sp_token - - # A trailing system message opens generation, while a system message - # followed by assistant opens that assistant history turn. - elif role in ["user", "developer"] or ( - role == "system" - and ( - index == len(messages) - 1 - or messages[index + 1].get("role") == "assistant" - ) - ): - # Normal generation: append Assistant + thinking token - prompt += ASSISTANT_SP_TOKEN - if not drop_thinking and thinking_mode == "thinking": - prompt += thinking_start_token - elif drop_thinking and thinking_mode == "thinking" and index >= last_user_idx: - prompt += thinking_start_token - else: - prompt += thinking_end_token - - return prompt - - -# ============================================================ -# Preprocessing -# ============================================================ - -def merge_tool_messages(messages: List[Dict[str, Any]]) -> List[Dict[str, Any]]: - """ - Merge tool messages into the preceding user message using content_blocks format. - - DeepSeek-V4 does not have a standalone "tool" role; instead, tool results - are encoded as blocks within user messages. - - This function converts a standard OpenAI-format conversation (with separate - "tool" role messages) into V4 format where tool results are merged into - user messages. - - Args: - messages: List of message dicts in OpenAI format. - - Returns: - Processed message list with tool messages merged into user messages. - """ - merged: List[Dict[str, Any]] = [] - - for msg in messages: - msg = copy.deepcopy(msg) - role = msg.get("role") - - if role == "tool": - # Convert tool message to a user message with tool_result block - tool_block = { - "type": "tool_result", - "tool_use_id": msg.get("tool_call_id", ""), - "content": msg.get("content", ""), - } - # Merge into previous message if it's already a user (merged tool) - if merged and merged[-1].get("role") == "user" and "content_blocks" in merged[-1]: - merged[-1]["content_blocks"].append(tool_block) - else: - merged.append({ - "role": "user", - "content_blocks": [tool_block], - }) - elif role == "user": - # Flatten OpenAI-style content blocks (images become placeholders) - text_content = flatten_content_blocks(msg.get("content", "")) - text_block = {"type": "text", "text": text_content} - if merged and merged[-1].get("role") == "user" and "content_blocks" in merged[-1] and merged[-1].get("task") is None: - merged[-1]["content_blocks"].append(text_block) - else: - new_msg = { - "role": "user", - "content": text_content, - "content_blocks": [text_block], - } - # Preserve extra fields (task, wo_eos, mask, etc.) - for key in ("task", "wo_eos", "mask"): - if key in msg: - new_msg[key] = msg[key] - merged.append(new_msg) - else: - merged.append(msg) - - return merged - - -def sort_tool_results_by_call_order(messages: List[Dict[str, Any]]) -> List[Dict[str, Any]]: - """ - Sort tool_result blocks within user messages by the order of tool_calls - in the preceding assistant message. - - Args: - messages: Preprocessed message list (after merge_tool_messages). - - Returns: - Message list with sorted tool result blocks. - """ - last_tool_call_order: Dict[str, int] = {} - - for msg in messages: - role = msg.get("role") - if role == "assistant" and msg.get("tool_calls"): - last_tool_call_order = {} - for idx, tc in enumerate(msg["tool_calls"]): - tc_id = tc.get("id") or tc.get("function", {}).get("id", "") - if tc_id: - last_tool_call_order[tc_id] = idx - - elif role == "user" and msg.get("content_blocks"): - tool_blocks = [b for b in msg["content_blocks"] if b.get("type") == "tool_result"] - if len(tool_blocks) > 1 and last_tool_call_order: - sorted_blocks = sorted( - tool_blocks, - key=lambda b: last_tool_call_order.get(b.get("tool_use_id", ""), 0) - ) - sorted_idx = 0 - new_blocks = [] - for block in msg["content_blocks"]: - if block.get("type") == "tool_result": - new_blocks.append(sorted_blocks[sorted_idx]) - sorted_idx += 1 - else: - new_blocks.append(block) - msg["content_blocks"] = new_blocks - - return messages - - -# ============================================================ -# Main Encoding Function -# ============================================================ - -def encode_messages( - messages: List[Dict[str, Any]], - thinking_mode: str, - context: Optional[List[Dict[str, Any]]] = None, - drop_thinking: bool = True, - add_default_bos_token: bool = True, - reasoning_effort: Optional[str] = None, -) -> str: - """ - Encode a list of messages into the DeepSeek-V4 prompt format. - - This is the main entry point for encoding conversations. It handles: - - BOS token insertion - - Thinking mode with optional reasoning content dropping - - Tool message merging into user messages - - Multi-turn conversation context - - Args: - messages: List of message dicts to encode. - thinking_mode: Either "chat" or "thinking". - context: Optional preceding context messages (already encoded prefix). - drop_thinking: If True, drop reasoning from earlier assistant turns - (only keep reasoning for messages after the last user message). - add_default_bos_token: Whether to prepend BOS token at conversation start. - reasoning_effort: Reasoning effort level, one of "low", "high", "max". - Only takes effect in thinking mode. None is treated as "low". - - Returns: - The encoded prompt string. - """ - context = context if context else [] - - # Preprocess: merge tool messages and sort tool results - messages = merge_tool_messages(messages) - messages = sort_tool_results_by_call_order(context + messages)[len(context):] - if context: - context = merge_tool_messages(context) - context = sort_tool_results_by_call_order(context) - - full_messages = context + messages - - prompt = bos_token if add_default_bos_token and len(context) == 0 else "" - - # Resolve drop_thinking: if any message has tools defined, don't drop thinking - effective_drop_thinking = drop_thinking - if any(m.get("tools") for m in full_messages): - effective_drop_thinking = False - - if thinking_mode == "thinking" and effective_drop_thinking: - full_messages = _drop_thinking_messages(full_messages) - # After dropping, recalculate how many messages to render - # (context may have shrunk too) - num_to_render = len(full_messages) - len(_drop_thinking_messages(context)) - context_len = len(full_messages) - num_to_render - else: - num_to_render = len(messages) - context_len = len(context) - - last_user_idx = find_last_user_index(full_messages) - - for idx in range(num_to_render): - prompt += render_message( - idx + context_len, - full_messages, - thinking_mode=thinking_mode, - drop_thinking=effective_drop_thinking, - reasoning_effort=reasoning_effort, - last_user_idx=last_user_idx, - ) - - return prompt - - -def _drop_thinking_messages(messages: List[Dict[str, Any]]) -> List[Dict[str, Any]]: - """ - Drop reasoning and non-essential messages before the last user message. - - Behavior: - - Messages with role in ["user", "system", "tool", "latest_reminder"] are always kept. - - Messages at or after the last user index are always kept. - - Assistant messages before the last user get reasoning removed. - - Developer messages before the last user are dropped entirely. - """ - last_user_idx = find_last_user_index(messages) - result = [] - keep_roles = {"user", "system", "tool", "latest_reminder", "direct_search_results"} - - for idx, msg in enumerate(messages): - role = msg.get("role") - if role in keep_roles or idx >= last_user_idx: - result.append(msg) - elif role == "assistant": - msg = copy.copy(msg) - msg.pop("reasoning", None) - result.append(msg) - # developer and other roles before last_user_idx are dropped - - return result - - -# fmt: on diff --git a/vllm/tokenizers/registry.py b/vllm/tokenizers/registry.py index a52bf18401..99a0dc4fca 100644 --- a/vllm/tokenizers/registry.py +++ b/vllm/tokenizers/registry.py @@ -45,11 +45,7 @@ _VLLM_TOKENIZERS = { # ``cohere`` mode uses the standard cached HF tokenizer; only the # renderer (template stage) is replaced with a melody-based one. "cohere": ("hf", "CachedHfTokenizer"), - "deepseek_v32": ("deepseek_v32", "DeepseekV32Tokenizer"), - "deepseek_v4": ("deepseek_v4", "DeepseekV4Tokenizer"), - "deepseek_v41": ("deepseek_v41", "DeepseekV41Tokenizer"), "hf": ("hf", "CachedHfTokenizer"), - "kimi_audio": ("kimi_audio", "KimiAudioTokenizer"), "kimi_k3": ("hf", "CachedHfTokenizer"), "mistral": ("mistral", "MistralTokenizer"), # Inkling uses the plain HF tokenizer for token operations; the "inkling" diff --git a/vllm/tool_parsers/AGENTS.md b/vllm/tool_parsers/AGENTS.md deleted file mode 100644 index 2bd487997c..0000000000 --- a/vllm/tool_parsers/AGENTS.md +++ /dev/null @@ -1,45 +0,0 @@ -# vllm/tool_parsers - -Every `--tool-call-parser` name resolves here. `ParserManager.get_parser` -composes the tool parser chosen by name with the reasoning parser into the -unified `Parser` the serving layer uses. The `ToolParser` interface itself is -legacy: it is kept so existing in-tree parsers and out-of-tree plugins keep -working, not as the base for new ones. - -## Layout - -- `abstract_tool_parser.py` — `ToolParser` (`extract_tool_calls`, - `extract_tool_calls_streaming`, `adjust_request`) and `ToolParserManager`. -- `__init__.py` — `_TOOL_PARSERS_TO_REGISTER`, the name → (file, class) table - that lazily registers every built-in parser. -- `structural_tag_registry.py` — structural-tag builders used to constrain - tool-call output for `tool_choice="required"` / named tools. -- `_tool_parser.py` — one file per family. Either a subclass of a - `ParserToolAdapter` generated from a `vllm/parser/.py` engine grammar - (`mistral_tool_parser.py`, `kimi_k2_tool_parser.py`; the `_engine_` suffix - is not applied consistently), or a self-contained `ToolParser` subclass. - -## Guidelines - -- Do not change the `ToolParser` interface in `abstract_tool_parser.py`. - Out-of-tree plugins loaded through `ToolParserManager.import_tool_parser` - subclass it. New capabilities go on `Parser` in `vllm/parser/`; if a legacy - parser needs the same capability, `DelegatingParser` supplies the default. -- For a new model, implement a unified `Parser` in `vllm/parser/` first, then - expose it here as a thin adapter subclass plus an entry in - `_TOOL_PARSERS_TO_REGISTER`. -- The `_TOOL_PARSERS_TO_REGISTER` key is what users pass on the command line; - treat renames as breaking. -- Keep adapters thin: registration, `adjust_request` tweaks, and class - attributes. Parsing logic belongs in `vllm/parser/`. -- Keep a model-specific fix inside that model's `_tool_parser.py`, or - its grammar in `vllm/parser/.py`, where possible. Touch - `abstract_tool_parser.py` or `structural_tag_registry.py` only for behavior - every parser needs, and say so in the PR. -- Set `structural_tag_model` only to a key in `SUPPORTED_STRUCTURAL_TAG_MODELS` - (an xgrammar builtin, or one registered with `register_vllm_structural_tag`). - It only takes effect under `VLLM_ENFORCE_STRICT_TOOL_CALLING`, where it also - disables the generic required/named handling for that parser. -- Comment only where the code alone would leave a reader with a question — - a request tweak in `adjust_request`, a marker handled unusually, a model - quirk being worked around. Keep it minimal and concise. diff --git a/vllm/tool_parsers/__init__.py b/vllm/tool_parsers/__init__.py index e8bccedde0..edfb9a5469 100644 --- a/vllm/tool_parsers/__init__.py +++ b/vllm/tool_parsers/__init__.py @@ -15,7 +15,6 @@ Register a lazy module mapping. Example: ToolParserManager.register_lazy_module( name="kimi_k2", - module_path="vllm.tool_parsers.kimi_k2_tool_parser", class_name="KimiK2ToolParser", ) """ diff --git a/vllm/tool_parsers/apertus_tool_parser.py b/vllm/tool_parsers/apertus_tool_parser.py deleted file mode 100644 index 35e65ccb7b..0000000000 --- a/vllm/tool_parsers/apertus_tool_parser.py +++ /dev/null @@ -1,553 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Tool call parser for Apertus models. - -Extracts tool calls from the format: -<|tools_prefix|>[{"function_name": {"arg1": "value1", ...}}, ...]<|tools_suffix|> - -Used when --enable-auto-tool-choice --tool-call-parser apertus are set. -""" - -import json -from collections.abc import Sequence - -import regex as re -from partial_json_parser.core.options import Allow - -from vllm.entrypoints.generate.base.protocol import ( - DeltaFunctionCall, - DeltaMessage, - DeltaToolCall, - ExtractedToolCallInformation, - FunctionCall, - ToolCall, -) -from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest -from vllm.entrypoints.openai.responses.protocol import ResponsesRequest -from vllm.logger import init_logger -from vllm.renderers.chat_utils import make_tool_call_id -from vllm.tokenizers import TokenizerLike -from vllm.tool_parsers.abstract_tool_parser import ( - Tool, - ToolParser, -) -from vllm.tool_parsers.utils import ( - find_common_prefix, - partial_json_loads, -) - -logger = init_logger(__name__) - -# Apertus special tokens for tool calls -TOOL_CALLS_PREFIX = "<|tools_prefix|>" -TOOL_CALLS_SUFFIX = "<|tools_suffix|>" - - -class ApertusToolParser(ToolParser): - """Tool call parser for Apertus models. - - Handles the extraction of tool calls from text in both non-streaming - (complete string) and streaming (chunked token) environments. - - The expected Apertus function call format is a JSON array of single-key dictionaries - sandwiched between special tokens: - `<|tools_prefix|>[{"function_name": {"arg1": "value1"}}, ...]<|tools_suffix|>` - - Examples: - >>> tokenizer = ... # Mock tokenizer - >>> parser = ApertusToolParser(tokenizer) - >>> output = 'I will check. <|tools_prefix|>[{"get_weather": '\ - '{"city": "Paris"}}]<|tools_suffix|>' - >>> request = ChatCompletionRequest(...) - >>> info = parser.extract_tool_calls(output, request) - >>> info.content - "I will check." - >>> info.tool_calls[0].function.name - "get_weather" - >>> info.tool_calls[0].function.arguments - '{"city": "Paris"}' - - """ - - def __init__(self, tokenizer: TokenizerLike, tools: list[Tool] | None = None): - """Initializes the ApertusToolParser. - - Args: - tokenizer: The model's tokenizer. - Must be provided to interact with special tokens. - tools: Optional list of tools available for the current request. - - Raises: - ValueError: If the `model_tokenizer` - is not successfully passed to the base class. - - """ - super().__init__(tokenizer, tools) - - if not self.model_tokenizer: - raise ValueError( - "The model tokenizer must be passed to the ToolParser " - "constructor during construction." - ) - # Regex to extract tool calls block (suffix is optional for incomplete outputs) - self.tool_call_regex = re.compile( - rf"{re.escape(TOOL_CALLS_PREFIX)}" - rf"(.*?)" - rf"(?:{re.escape(TOOL_CALLS_SUFFIX)}|$)", - re.DOTALL, - ) - - self._reset_streaming_state() - - def _reset_streaming_state(self) -> None: - """Resets all streaming state variables for a new completion request. - - This clears the delta text buffer and resets the pointers used to - track the currently streaming tool index and arguments. Called implicitly - during initialization and should be called between separate streams. - """ - self.buffered_delta_text = "" - self.current_tool_id = -1 - self.current_tool_name_sent = False - self.streamed_args_for_tool: list[str] = [] - - def adjust_request( - self, request: ChatCompletionRequest | ResponsesRequest - ) -> ChatCompletionRequest | ResponsesRequest: - """Adjusts the generation request to ensure special tool tokens are not skipped. - - Forces `skip_special_tokens=False` if tools are actively being evaluated, - ensuring the tools special tokens are surfaced to the engine for parsing. - - Args: - request: The incoming OpenAI-compatible chat completion request. - - Returns: - The potentially modified chat completion request. - - """ - request = super().adjust_request(request) - if request.tools and request.tool_choice != "none": - request.skip_special_tokens = False - return request - - def _buffer_delta_text(self, delta_text: str) -> str: - """Buffers incoming delta chunks to prevent - fragmentation of multi-token special tags. - - If a chunk ends with a partial match of - `<|tools_prefix|>` or `<|tools_suffix|>`, - it holds that part back until the next chunk clarifies if it's the actual tag - or just normal text. - - Args: - delta_text: The newly generated text chunk - - Returns: - The safe, verified text chunk free of partial tag collisions. - - Examples: - >>> parser = ApertusToolParser(...) - >>> parser._buffer_delta_text("Let me check <|tool" \ - "Let me check " # "<|tool" is buffered internally - >>> parser._buffer_delta_text("s_prefix|>" \ - "<|tools_prefix|>" # Buffer released on completion - - """ - self.buffered_delta_text += delta_text - text = self.buffered_delta_text - - for tag in (TOOL_CALLS_PREFIX, TOOL_CALLS_SUFFIX): - if text.endswith(tag): - self.buffered_delta_text = "" - return text - - # Evaluate longest possible partial match first - for i in range(len(tag) - 1, 0, -1): - if text.endswith(tag[:i]): - self.buffered_delta_text = text[-i:] - return text[:-i] - - self.buffered_delta_text = "" - return text - - def extract_tool_calls( - self, - model_output: str, - request: ChatCompletionRequest, - ) -> ExtractedToolCallInformation: - """Extract tool calls from a complete model response (non-streaming). - - Args: - model_output: The full completion string generated by the model. - request: The current chat completion - request context containing tool schemas. - - Returns: - An `ExtractedToolCallInformation` object containing normal text content - and a list of fully formatted `ToolCall` objects. - - Examples: - >>> output = 'Let me see. <|tools_prefix|>[{"get_weather":' \ - '{"loc": "Paris"}}]<|tools_suffix|>' - >>> info = parser.extract_tool_calls(output, request) - >>> info.tools_called - True - >>> info.content - 'Let me see.' - >>> info.tool_calls[0].function.name - 'get_weather' - - """ - match = self.tool_call_regex.search(model_output) - if not match: - return ExtractedToolCallInformation( - tools_called=False, tool_calls=[], content=model_output - ) - - try: - # group(1) might contain trailing text if the suffix is missing - matched_text = match.group(1) - stripped_text = matched_text.lstrip() - - try: - # Use raw_decode to robustly isolate - # the valid JSON array from any trailing garbage - parsed_json, idx = json.JSONDecoder().raw_decode(stripped_text) - trailing_in_group = stripped_text[idx:] - except json.JSONDecodeError: - # Fallback sequentially to partial parser for token-truncated requests - parsed_json, _ = partial_json_loads(matched_text, Allow.ALL) - trailing_in_group = "" - - if not isinstance(parsed_json, list): - parsed_json = [parsed_json] if parsed_json else [] - - tool_calls: list[ToolCall] = [] - for obj in parsed_json: - if isinstance(obj, dict) and obj: - name, args = next(iter(obj.items())) - tool_calls.append( - ToolCall( - type="function", - id=make_tool_call_id(), - function=FunctionCall( - name=name, - arguments=json.dumps(args, ensure_ascii=False), - ), - ) - ) - - # Content combines any generated text - # prior to and safely after the tool block - content_str = model_output[: match.start()].strip() - - # Surface any hallucinated text inside - # the regex group (due to missing suffix) - if trailing_in_group.strip(): - trailing = trailing_in_group.replace(TOOL_CALLS_SUFFIX, "").strip() - if trailing: - content_str = (content_str + "\n" + trailing).strip() - - # Surface text natively generated after the explicit suffix - after_suffix = ( - model_output[match.end() :].replace(TOOL_CALLS_SUFFIX, "").strip() - ) - if after_suffix: - content_str = (content_str + "\n" + after_suffix).strip() - - return ExtractedToolCallInformation( - tools_called=True, - tool_calls=tool_calls, - content=content_str if content_str else None, - ) - - except Exception: - logger.exception("Error extracting tool calls from Apertus response") - return ExtractedToolCallInformation( - tools_called=False, tool_calls=[], content=model_output - ) - - def extract_tool_calls_streaming( - self, - previous_text: str, - current_text: str, - delta_text: str, - previous_token_ids: Sequence[int], - current_token_ids: Sequence[int], - delta_token_ids: Sequence[int], - request: ChatCompletionRequest, - ) -> DeltaMessage | None: - """Handles streaming chunks. - - Args: - previous_text: The complete model text generated prior to this chunk. - current_text: The complete model text including this chunk. - delta_text: The incremental text addition. - previous_token_ids: Tokens generated prior to this chunk. - current_token_ids: Total tokens generated. - delta_token_ids: Incremental token additions. - request: The chat completion request. - - Returns: - A `DeltaMessage` with updated content or tool argument diffs, or `None` if - the chunk shouldn't emit visible changes yet (e.g. it was purely buffered). - - Examples: - >>> prev = '<|tools_prefix|>[{"get_weather": {"loc' - >>> cur = '<|tools_prefix|>[{"get_weather": {"location": "Paris"}}' - >>> delta = 'ation": "Paris"}}' - >>> msg = parser.extract_tool_calls_streaming( - ... prev, cur, delta, ..., request - ... ) - >>> msg.tool_calls[0].function.arguments - 'ation": "Paris"}' - - """ - delta_text = self._buffer_delta_text(delta_text) - if not delta_text: - return None - - # Fast path: normal text generation before any tools are invoked - if TOOL_CALLS_PREFIX not in current_text: - return DeltaMessage(content=delta_text) - - try: - return self._extract_streaming(current_text, delta_text) - except Exception: - logger.exception("Error in Apertus streaming tool call extraction") - return None - - def _extract_streaming( - self, current_text: str, delta_text: str - ) -> DeltaMessage | None: - """Core streaming logic. - Separates visible chat text from JSON blocks and computes diffs. - - Args: - current_text: The full generated output string so far. - delta_text: The latest chunk of text added. - - Returns: - A `DeltaMessage` containing the `content` delta and/or `tool_calls` delta. - - """ - prefix_idx = current_text.rfind(TOOL_CALLS_PREFIX) - suffix_idx = current_text.rfind(TOOL_CALLS_SUFFIX) - - is_inside_tools = prefix_idx > suffix_idx - - json_completed = False - json_end_idx: int | None = None - - # Check if the JSON array successfully closed implicitly - if is_inside_tools: - json_start = prefix_idx + len(TOOL_CALLS_PREFIX) - s = current_text[json_start:].lstrip() - try: - # If raw_decode succeeds, - # the JSON array is fully formed and implicitly closed - _, idx = json.JSONDecoder().raw_decode(s) - json_end_idx = len(current_text) - len(s) + idx - json_completed, is_inside_tools = True, False - except Exception: - pass - - just_finished = (TOOL_CALLS_SUFFIX in delta_text) or json_completed - - # 1. Fast path: Output normal text immediately - # if we are completely outside tool block constraints - if not is_inside_tools and not just_finished: - text = delta_text.replace(TOOL_CALLS_PREFIX, "").replace( - TOOL_CALLS_SUFFIX, "" - ) - return DeltaMessage(content=text) if text else None - - # 2. Extract leading and trailing normal text directly adjacent to tool blocks - content_str = "" - if TOOL_CALLS_PREFIX in delta_text: - content_str += delta_text.split(TOOL_CALLS_PREFIX)[0].replace( - TOOL_CALLS_SUFFIX, "" - ) - - if just_finished: - if json_completed and json_end_idx is not None: - # The tool block finished in this chunk via implicit JSON completion - # Ensure we strictly isolate - # and extract only trailing text that is part of `delta_text` - delta_start_idx = len(current_text) - len(delta_text) - content_start = max(json_end_idx, delta_start_idx) - if content_start < len(current_text): - content_str += current_text[content_start:].replace( - TOOL_CALLS_SUFFIX, "" - ) - else: - content_str += delta_text.split(TOOL_CALLS_SUFFIX)[-1] - - # 3. Extract the isolated JSON array string for the active block - json_start = prefix_idx + len(TOOL_CALLS_PREFIX) - json_end = suffix_idx if suffix_idx > prefix_idx else json_end_idx - json_str = current_text[json_start:json_end] - - tool_calls = self._parse_and_diff_json(json_str, is_final=not is_inside_tools) - - if tool_calls or content_str: - return DeltaMessage( - content=content_str if content_str else None, - tool_calls=tool_calls if tool_calls else None, - ) - - return None - - def _parse_and_diff_json( - self, json_str: str, is_final: bool - ) -> list[DeltaToolCall]: - """Parses an isolated, potentially incomplete streaming JSON array and returns - newly accumulated tool call diffs. - - Args: - json_str: The extracted JSON array string so far - (e.g. `[{"weather": {"city": "Par"}]`). - is_final: True if the tool block has received its closing`<|tools_suffix|>` - - Returns: - A list of `DeltaToolCall` - items representing string diffs in function arguments - to stream back to the client. - - """ - try: - # An exponent can change a number's already-streamed representation. - flags = Allow.ALL if is_final else Allow.ALL & ~Allow.NUM - parsed, _ = partial_json_loads(json_str, flags) - if not isinstance(parsed, list): - parsed = [parsed] if parsed else [] - except Exception: - return [] - - if not parsed: - return [] - - tool_calls: list[DeltaToolCall] = [] - latest_index = len(parsed) - 1 - - # Catch up and finalize any tools we fully skipped over in one large text delta - while self.current_tool_id < latest_index: - if self.current_tool_id >= 0: - if not self.current_tool_name_sent: - self._emit_tool_name(parsed, self.current_tool_id, tool_calls) - - delta = self._get_tool_diff(parsed, self.current_tool_id, is_final=True) - if delta: - tool_calls.append(delta) - - self.current_tool_id += 1 - self.current_tool_name_sent = False - while len(self.streamed_args_for_tool) <= self.current_tool_id: - self.streamed_args_for_tool.append("") - - # Stream the currently active tool - if self.current_tool_id >= 0: - if not self.current_tool_name_sent: - self._emit_tool_name(parsed, self.current_tool_id, tool_calls) - - delta = self._get_tool_diff(parsed, self.current_tool_id, is_final) - if delta: - tool_calls.append(delta) - - return tool_calls - - def _emit_tool_name( - self, parsed: list, index: int, tool_calls: list[DeltaToolCall] - ) -> None: - """Extracts and emits the function name mapped to a new tool call ID. - - Args: - parsed: The partially parsed JSON list containing tool dictionaries. - index: The active index within the JSON list. - tool_calls: The running list of delta chunks to mutate. - - Examples: - Appends `DeltaToolCall(index=0, - function=DeltaFunctionCall(name="get_weather", ...))` - to the `tool_calls` list and marks the name as sent. - - """ - obj = parsed[index] - if isinstance(obj, dict) and obj: - name = next(iter(obj)) - self.current_tool_name_sent = True - tool_calls.append( - DeltaToolCall( - index=index, - type="function", - id=make_tool_call_id(), - function=DeltaFunctionCall(name=name, arguments="").model_dump( - exclude_none=True - ), - ) - ) - - def _get_tool_diff( - self, parsed: list, index: int, is_final: bool - ) -> DeltaToolCall | None: - """Calculates the exact string difference to safely append new tool parameters. - - This ensures characters like `{`, `}`, and `"` don't jump around unevenly - in the UI frontend while streaming incomplete JSON arguments. - - Args: - parsed: The latest list of parsed JSON objects. - index: The active tool's array index. - is_final: Whether to emit - trailing structural brackets (True if block is done). - - Returns: - A `DeltaToolCall` mapping to the arguments diff, - or None if no text was appended. - - Examples: - >>> # Previous streamed state: '{"city": "Pari' - >>> # Current full parse state: '{"city": "Paris"}' - >>> # Returns diff (closing bracket suppressed until final): - >>> parser._get_tool_diff(parsed, index=0, is_final=False) - DeltaToolCall(index=0, function=DeltaFunctionCall(arguments='s')) - - """ - obj = parsed[index] - if not isinstance(obj, dict) or not obj: - return None - - name, args = next(iter(obj.items())) - if args is None: - return None - - args_json = json.dumps(args, ensure_ascii=False) - - # Suppress trailing structural characters - # during stream (looks cleaner in frontends) - if not is_final: - while args_json and args_json[-1] in ("}", '"', "]", " ", ","): - args_json = args_json[:-1] - - prev_sent = self.streamed_args_for_tool[index] - if args_json == prev_sent: - return None - - prefix = find_common_prefix(prev_sent, args_json) - if len(prefix) < len(prev_sent): - # Backtrack state if partial parser structurally updates a past assumption - self.streamed_args_for_tool[index] = prefix - return None - - diff = args_json[len(prev_sent) :] - if diff: - self.streamed_args_for_tool[index] = args_json - return DeltaToolCall( - index=index, - function=DeltaFunctionCall(arguments=diff).model_dump( - exclude_none=True - ), - ) - - return None diff --git a/vllm/tool_parsers/beam_tool_parser.py b/vllm/tool_parsers/beam_tool_parser.py deleted file mode 100644 index dbfd820688..0000000000 --- a/vllm/tool_parsers/beam_tool_parser.py +++ /dev/null @@ -1,6 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from vllm.parser.beam import BeamToolParser - -__all__ = ["BeamToolParser"] diff --git a/vllm/tool_parsers/cohere_command_tool_parser.py b/vllm/tool_parsers/cohere_command_tool_parser.py deleted file mode 100644 index eca5c54967..0000000000 --- a/vllm/tool_parsers/cohere_command_tool_parser.py +++ /dev/null @@ -1,18 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from typing import ClassVar - -from vllm.tool_parsers import ToolParser - - -class BaseCohereCommandToolParser(ToolParser): - melody_preset: ClassVar[str] - - -class CohereCommand3ToolParser(BaseCohereCommandToolParser): - melody_preset = "cmd3" - - -class CohereCommand4ToolParser(BaseCohereCommandToolParser): - melody_preset = "cmd4" diff --git a/vllm/tool_parsers/deepseekv31_tool_parser.py b/vllm/tool_parsers/deepseekv31_tool_parser.py deleted file mode 100644 index ff5f605f51..0000000000 --- a/vllm/tool_parsers/deepseekv31_tool_parser.py +++ /dev/null @@ -1,392 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from collections.abc import Sequence - -import regex as re - -from vllm.entrypoints.generate.base.protocol import ( - DeltaFunctionCall, - DeltaMessage, - DeltaToolCall, - ExtractedToolCallInformation, - FunctionCall, - ToolCall, -) -from vllm.entrypoints.openai.chat_completion.protocol import ( - ChatCompletionRequest, -) -from vllm.logger import init_logger -from vllm.renderers.chat_utils import make_tool_call_id -from vllm.tokenizers import TokenizerLike -from vllm.tool_parsers.abstract_tool_parser import Tool, ToolParser - -logger = init_logger(__name__) - - -class DeepSeekV31ToolParser(ToolParser): - structural_tag_model = "deepseek_v3_1" - - def __init__(self, tokenizer: TokenizerLike, tools: list[Tool] | None = None): - super().__init__(tokenizer, tools) - - self.current_tool_name_sent: bool = False - self.prev_tool_call_arr: list[dict] = [] - self.current_tool_id: int = -1 - self.streamed_args_for_tool: list[ - str - ] = [] # map what has been streamed for each tool so far to a list - - self.tool_calls_start_token: str = "<|tool▁calls▁begin|>" - self.tool_calls_end_token: str = "<|tool▁calls▁end|>" - - self.tool_call_start_token: str = "<|tool▁call▁begin|>" - self.tool_call_end_token: str = "<|tool▁call▁end|>" - - self.tool_call_regex = re.compile( - r"<|tool▁call▁begin|>(?P.*?)<|tool▁sep|>(?P.*?)<|tool▁call▁end|>" - ) - - self.stream_tool_call_portion_regex = re.compile( - r"(?P.*)<|tool▁sep|>(?P.*)" - ) - - self.stream_tool_call_name_regex = re.compile( - r"(?P.*)<|tool▁sep|>" - ) - - if not self.model_tokenizer: - raise ValueError( - "The model tokenizer must be passed to the ToolParser " - "constructor during construction." - ) - self.tool_calls_start_token_id = self.vocab.get(self.tool_calls_start_token) - self.tool_calls_end_token_id = self.vocab.get(self.tool_calls_end_token) - - self.tool_call_start_token_id = self.vocab.get(self.tool_call_start_token) - self.tool_call_end_token_id = self.vocab.get(self.tool_call_end_token) - - if ( - self.tool_calls_start_token_id is None - or self.tool_calls_end_token_id is None - ): - raise RuntimeError( - "DeepSeek-V3.1 Tool parser could not locate tool call " - "start/end tokens in the tokenizer!" - ) - - def extract_tool_calls( - self, - model_output: str, - request: ChatCompletionRequest, - ) -> ExtractedToolCallInformation: - # sanity check; avoid unnecessary processing - if self.tool_calls_start_token not in model_output: - return ExtractedToolCallInformation( - tools_called=False, tool_calls=[], content=model_output - ) - - else: - try: - # there are two possible captures - between tags, or between a - # tag and end-of-string so the result of - # findall is an array of tuples where one is a function call and - # the other is None - function_call_tuples = self.tool_call_regex.findall(model_output) - - tool_calls = [] - for match in function_call_tuples: - function_name, function_args = match - tool_calls.append( - ToolCall( - type="function", - function=FunctionCall( - name=function_name, arguments=function_args - ), - ) - ) - - content = model_output[: model_output.find(self.tool_calls_start_token)] - return ExtractedToolCallInformation( - tools_called=True, - tool_calls=tool_calls, - content=content if content else None, - ) - - except Exception: - logger.exception("Error in extracting tool call from response.") - return ExtractedToolCallInformation( - tools_called=False, tool_calls=[], content=model_output - ) - - def extract_tool_calls_streaming( - self, - previous_text: str, - current_text: str, - delta_text: str, - previous_token_ids: Sequence[int], - current_token_ids: Sequence[int], - delta_token_ids: Sequence[int], - request: ChatCompletionRequest, - ) -> DeltaMessage | None: - logger.debug("delta_text: %s", delta_text) - logger.debug("delta_token_ids: %s", delta_token_ids) - # check to see if we should be streaming a tool call - is there a - if self.tool_calls_start_token_id not in current_token_ids: - logger.debug("No tool call tokens found!") - return DeltaMessage(content=delta_text) - delta_text = delta_text.replace(self.tool_calls_start_token, "").replace( - self.tool_calls_end_token, "" - ) - try: - # figure out where we are in the parsing by counting tool call - # start & end tags - prev_tool_start_count = previous_token_ids.count( - self.tool_call_start_token_id - ) - prev_tool_end_count = previous_token_ids.count(self.tool_call_end_token_id) - cur_tool_start_count = current_token_ids.count( - self.tool_call_start_token_id - ) - cur_tool_end_count = current_token_ids.count(self.tool_call_end_token_id) - tool_call_portion = None - text_portion = None - - # case: if we're generating text, OR rounding out a tool call - if ( - cur_tool_start_count == cur_tool_end_count - and prev_tool_end_count == cur_tool_end_count - and self.tool_call_end_token not in delta_text - ): - logger.debug("Generating text content! skipping tool parsing.") - return DeltaMessage(content=delta_text) - - if self.tool_call_end_token in delta_text: - logger.debug("tool_call_end_token in delta_text") - full_text = current_text + delta_text - tool_call_portion = ( - full_text.split(self.tool_call_start_token)[-1] - .split(self.tool_call_end_token)[0] - .rstrip() - ) - delta_text = delta_text.split(self.tool_call_end_token)[0].rstrip() - text_portion = delta_text.split(self.tool_call_end_token)[-1].lstrip() - - # case -- we're starting a new tool call - if ( - cur_tool_start_count > cur_tool_end_count - and cur_tool_start_count > prev_tool_start_count - ): - if len(delta_token_ids) > 1: - tool_call_portion = current_text.split(self.tool_call_start_token)[ - -1 - ] - else: - tool_call_portion = None - delta = None - - text_portion = None - - # set cursors and state appropriately - self.current_tool_id += 1 - self.current_tool_name_sent = False - self.streamed_args_for_tool.append("") - logger.debug("Starting on a new tool %s", self.current_tool_id) - - # case -- we're updating an existing tool call - elif ( - cur_tool_start_count > cur_tool_end_count - and cur_tool_start_count == prev_tool_start_count - ): - # get the portion of the text that's the tool call - tool_call_portion = current_text.split(self.tool_call_start_token)[-1] - text_portion = None - - # case -- the current tool call is being closed. - elif ( - cur_tool_start_count == cur_tool_end_count - and cur_tool_end_count >= prev_tool_end_count - ): - if self.prev_tool_call_arr is None or len(self.prev_tool_call_arr) == 0: - logger.debug("attempting to close tool call, but no tool call") - return None - diff = self.prev_tool_call_arr[self.current_tool_id].get("arguments") - if diff: - diff = ( - diff.encode("utf-8").decode("unicode_escape") - if diff is str - else diff - ) - if '"}' not in delta_text: - return None - end_loc = delta_text.rindex('"}') - diff = delta_text[:end_loc] + '"}' - logger.debug( - "Finishing tool and found diff that had not " - "been streamed yet: %s", - diff, - ) - self.streamed_args_for_tool[self.current_tool_id] += diff - return DeltaMessage( - tool_calls=[ - DeltaToolCall( - index=self.current_tool_id, - function=DeltaFunctionCall(arguments=diff).model_dump( - exclude_none=True - ), - ) - ] - ) - - # case -- otherwise we're just generating text - else: - text = delta_text.replace(self.tool_call_start_token, "") - text = text.replace(self.tool_call_end_token, "") - delta = DeltaMessage(tool_calls=[], content=text) - return delta - - current_tool_call = dict() - if tool_call_portion: - current_tool_call_matches = self.stream_tool_call_portion_regex.match( - tool_call_portion - ) - if current_tool_call_matches: - tool_name, tool_args = current_tool_call_matches.groups() - current_tool_call["name"] = tool_name - current_tool_call["arguments"] = tool_args - else: - current_tool_call_name_matches = ( - self.stream_tool_call_name_regex.match(tool_call_portion) - ) - if current_tool_call_name_matches: - tool_name = current_tool_call_name_matches.groups() - current_tool_call["name"] = tool_name - current_tool_call["arguments"] = "" - else: - logger.debug("Not enough token") - return None - - # case - we haven't sent the tool name yet. If it's available, send - # it. otherwise, wait until it's available. - if not self.current_tool_name_sent: - if current_tool_call is None: - return None - function_name: str | None = current_tool_call.get("name") - if function_name: - self.current_tool_name_sent = True - return DeltaMessage( - tool_calls=[ - DeltaToolCall( - index=self.current_tool_id, - type="function", - id=make_tool_call_id(), - function=DeltaFunctionCall( - name=function_name - ).model_dump(exclude_none=True), - ) - ] - ) - else: - return None - - # case -- otherwise, send the tool call delta - - # if the tool call portion is None, send the delta as text - if tool_call_portion is None: - # if there's text but not tool calls, send that - - # otherwise None to skip chunk - delta = ( - DeltaMessage(content=delta_text) - if text_portion is not None - else None - ) - return delta - - # now, the nitty-gritty of tool calls - # now we have the portion to parse as tool call. - - logger.debug( - "Trying to parse current tool call with ID %s", self.current_tool_id - ) - - # if we're starting a new tool call, push an empty object in as - # a placeholder for the arguments - if len(self.prev_tool_call_arr) <= self.current_tool_id: - self.prev_tool_call_arr.append({}) - - # main logic for tool parsing here - compare prev. partially-parsed - # JSON to the current partially-parsed JSON - prev_arguments = self.prev_tool_call_arr[self.current_tool_id].get( - "arguments" - ) - cur_arguments = current_tool_call.get("arguments") - - logger.debug("diffing old arguments: %s", prev_arguments) - logger.debug("against new ones: %s", cur_arguments) - - # case -- no arguments have been created yet. skip sending a delta. - if not cur_arguments and not prev_arguments: - logger.debug("Skipping text %s - no arguments", delta_text) - delta = None - - # case -- prev arguments are defined, but non are now. - # probably impossible, but not a fatal error - just keep going - elif not cur_arguments and prev_arguments: - logger.error( - "should be impossible to have arguments reset " - "mid-call. skipping streaming anything." - ) - delta = None - - # case -- we now have the first info about arguments available from - # autocompleting the JSON - elif cur_arguments and not prev_arguments: - delta = DeltaMessage( - tool_calls=[ - DeltaToolCall( - index=self.current_tool_id, - function=DeltaFunctionCall( - arguments=cur_arguments - ).model_dump(exclude_none=True), - ) - ] - ) - self.streamed_args_for_tool[self.current_tool_id] = cur_arguments - - # last case -- we have an update to existing arguments. - elif cur_arguments and prev_arguments: - if ( - isinstance(delta_text, str) - and cur_arguments != prev_arguments - and len(cur_arguments) > len(prev_arguments) - and cur_arguments.startswith(prev_arguments) - ): - delta_arguments = cur_arguments[len(prev_arguments) :] - logger.debug("got diff %s", delta_text) - - delta = DeltaMessage( - tool_calls=[ - DeltaToolCall( - index=self.current_tool_id, - function=DeltaFunctionCall( - arguments=delta_arguments - ).model_dump(exclude_none=True), - ) - ] - ) - self.streamed_args_for_tool[self.current_tool_id] = cur_arguments - else: - delta = None - - # handle saving the state for the current tool into - # the "prev" list for use in diffing for the next iteration - if self.current_tool_id == len(self.prev_tool_call_arr) - 1: - self.prev_tool_call_arr[self.current_tool_id] = current_tool_call - else: - self.prev_tool_call_arr.append(current_tool_call) - - return delta - - except Exception: - logger.exception("Error trying to handle streaming tool call.") - return None # do not stream a delta. skip this token ID. diff --git a/vllm/tool_parsers/deepseekv32_engine_tool_parser.py b/vllm/tool_parsers/deepseekv32_engine_tool_parser.py deleted file mode 100644 index 4747fd10f3..0000000000 --- a/vllm/tool_parsers/deepseekv32_engine_tool_parser.py +++ /dev/null @@ -1,8 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from vllm.parser.engine.registered_adapters import DeepSeekV32ParserToolAdapter - - -class DeepSeekV32EngineToolParser(DeepSeekV32ParserToolAdapter): # type: ignore[valid-type, misc] - structural_tag_model = "deepseek_v3_2" diff --git a/vllm/tool_parsers/deepseekv3_tool_parser.py b/vllm/tool_parsers/deepseekv3_tool_parser.py deleted file mode 100644 index 649eded287..0000000000 --- a/vllm/tool_parsers/deepseekv3_tool_parser.py +++ /dev/null @@ -1,395 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from collections.abc import Sequence - -import regex as re - -from vllm.entrypoints.generate.base.protocol import ( - DeltaFunctionCall, - DeltaMessage, - DeltaToolCall, - ExtractedToolCallInformation, - FunctionCall, - ToolCall, -) -from vllm.entrypoints.openai.chat_completion.protocol import ( - ChatCompletionRequest, -) -from vllm.logger import init_logger -from vllm.renderers.chat_utils import make_tool_call_id -from vllm.tokenizers import TokenizerLike -from vllm.tool_parsers.abstract_tool_parser import ( - Tool, - ToolParser, -) - -logger = init_logger(__name__) - - -class DeepSeekV3ToolParser(ToolParser): - structural_tag_model = "deepseek_r1" - - def __init__(self, tokenizer: TokenizerLike, tools: list[Tool] | None = None): - super().__init__(tokenizer, tools) - - self.current_tool_name_sent: bool = False - self.prev_tool_call_arr: list[dict] = [] - self.current_tool_id: int = -1 - self.streamed_args_for_tool: list[ - str - ] = [] # map what has been streamed for each tool so far to a list - - self.tool_calls_start_token: str = "<|tool▁calls▁begin|>" - self.tool_calls_end_token: str = "<|tool▁calls▁end|>" - - self.tool_call_start_token: str = "<|tool▁call▁begin|>" - self.tool_call_end_token: str = "<|tool▁call▁end|>" - - self.tool_call_regex = re.compile( - r"<|tool▁call▁begin|>(?P.*)<|tool▁sep|>(?P.*)\n```json\n(?P.*)\n```<|tool▁call▁end|>" - ) - - self.stream_tool_call_portion_regex = re.compile( - r"(?P.*)<|tool▁sep|>(?P.*)\n```json\n(?P.*[^\n`])" - ) - - self.stream_tool_call_name_regex = re.compile( - r"(?P.*)<|tool▁sep|>(?P.*)\n" - ) - - if not self.model_tokenizer: - raise ValueError( - "The model tokenizer must be passed to the ToolParser " - "constructor during construction." - ) - self.tool_calls_start_token_id = self.vocab.get(self.tool_calls_start_token) - self.tool_calls_end_token_id = self.vocab.get(self.tool_calls_end_token) - - self.tool_call_start_token_id = self.vocab.get(self.tool_call_start_token) - self.tool_call_end_token_id = self.vocab.get(self.tool_call_end_token) - - if ( - self.tool_calls_start_token_id is None - or self.tool_calls_end_token_id is None - ): - raise RuntimeError( - "DeepSeek-V3 Tool parser could not locate tool call start/end " - "tokens in the tokenizer!" - ) - - def extract_tool_calls( - self, - model_output: str, - request: ChatCompletionRequest, - ) -> ExtractedToolCallInformation: - # sanity check; avoid unnecessary processing - if self.tool_calls_start_token not in model_output: - return ExtractedToolCallInformation( - tools_called=False, tool_calls=[], content=model_output - ) - - else: - try: - # there are two possible captures - between tags, or between a - # tag and end-of-string so the result of - # findall is an array of tuples where one is a function call and - # the other is None - function_call_tuples = self.tool_call_regex.findall(model_output) - - tool_calls = [] - for match in function_call_tuples: - tool_type, function_name, function_args = match - tool_calls.append( - ToolCall( - type=tool_type, - function=FunctionCall( - name=function_name, arguments=function_args - ), - ) - ) - - content = model_output[: model_output.find(self.tool_calls_start_token)] - return ExtractedToolCallInformation( - tools_called=True, - tool_calls=tool_calls, - content=content if content else None, - ) - - except Exception: - logger.exception("Error in extracting tool call from response.") - return ExtractedToolCallInformation( - tools_called=False, tool_calls=[], content=model_output - ) - - def extract_tool_calls_streaming( - self, - previous_text: str, - current_text: str, - delta_text: str, - previous_token_ids: Sequence[int], - current_token_ids: Sequence[int], - delta_token_ids: Sequence[int], - request: ChatCompletionRequest, - ) -> DeltaMessage | None: - logger.debug("delta_text: %s", delta_text) - logger.debug("delta_token_ids: %s", delta_token_ids) - # check to see if we should be streaming a tool call - is there a - if self.tool_calls_start_token_id not in current_token_ids: - logger.debug("No tool call tokens found!") - return DeltaMessage(content=delta_text) - delta_text = delta_text.replace(self.tool_calls_start_token, "").replace( - self.tool_calls_end_token, "" - ) - try: - # figure out where we are in the parsing by counting tool call - # start & end tags - prev_tool_start_count = previous_token_ids.count( - self.tool_call_start_token_id - ) - prev_tool_end_count = previous_token_ids.count(self.tool_call_end_token_id) - cur_tool_start_count = current_token_ids.count( - self.tool_call_start_token_id - ) - cur_tool_end_count = current_token_ids.count(self.tool_call_end_token_id) - tool_call_portion = None - text_portion = None - - # case: if we're generating text, OR rounding out a tool call - if ( - cur_tool_start_count == cur_tool_end_count - and prev_tool_end_count == cur_tool_end_count - and self.tool_call_end_token not in delta_text - ): - logger.debug("Generating text content! skipping tool parsing.") - return DeltaMessage(content=delta_text) - - if self.tool_call_end_token in delta_text: - logger.debug("tool_call_end_token in delta_text") - full_text = current_text + delta_text - tool_call_portion = ( - full_text.split(self.tool_call_start_token)[-1] - .split(self.tool_call_end_token)[0] - .rstrip() - ) - delta_text = delta_text.split(self.tool_call_end_token)[0].rstrip() - text_portion = delta_text.split(self.tool_call_end_token)[-1].lstrip() - - # case -- we're starting a new tool call - if ( - cur_tool_start_count > cur_tool_end_count - and cur_tool_start_count > prev_tool_start_count - ): - if len(delta_token_ids) > 1: - tool_call_portion = current_text.split(self.tool_call_start_token)[ - -1 - ] - else: - tool_call_portion = None - delta = None - - text_portion = None - - # set cursors and state appropriately - self.current_tool_id += 1 - self.current_tool_name_sent = False - self.streamed_args_for_tool.append("") - logger.debug("Starting on a new tool %s", self.current_tool_id) - - # case -- we're updating an existing tool call - elif ( - cur_tool_start_count > cur_tool_end_count - and cur_tool_start_count == prev_tool_start_count - ): - # get the portion of the text that's the tool call - tool_call_portion = current_text.split(self.tool_call_start_token)[-1] - text_portion = None - - # case -- the current tool call is being closed. - elif ( - cur_tool_start_count == cur_tool_end_count - and cur_tool_end_count >= prev_tool_end_count - ): - if self.prev_tool_call_arr is None or len(self.prev_tool_call_arr) == 0: - logger.debug("attempting to close tool call, but no tool call") - return None - diff = self.prev_tool_call_arr[self.current_tool_id].get("arguments") - if diff: - diff = ( - diff.encode("utf-8").decode("unicode_escape") - if diff is str - else diff - ) - if '"}' not in delta_text: - return None - end_loc = delta_text.rindex('"}') - diff = delta_text[:end_loc] + '"}' - logger.debug( - "Finishing tool and found diff that had not " - "been streamed yet: %s", - diff, - ) - self.streamed_args_for_tool[self.current_tool_id] += diff - return DeltaMessage( - tool_calls=[ - DeltaToolCall( - index=self.current_tool_id, - function=DeltaFunctionCall(arguments=diff).model_dump( - exclude_none=True - ), - ) - ] - ) - - # case -- otherwise we're just generating text - else: - text = delta_text.replace(self.tool_call_start_token, "") - text = text.replace(self.tool_call_end_token, "") - delta = DeltaMessage(tool_calls=[], content=text) - return delta - - current_tool_call = dict() - if tool_call_portion: - current_tool_call_matches = self.stream_tool_call_portion_regex.match( - tool_call_portion - ) - if current_tool_call_matches: - tool_type, tool_name, tool_args = current_tool_call_matches.groups() - current_tool_call["name"] = tool_name - current_tool_call["arguments"] = tool_args - else: - current_tool_call_name_matches = ( - self.stream_tool_call_name_regex.match(tool_call_portion) - ) - if current_tool_call_name_matches: - tool_type, tool_name = current_tool_call_name_matches.groups() - current_tool_call["name"] = tool_name - current_tool_call["arguments"] = "" - else: - logger.debug("Not enough token") - return None - - # case - we haven't sent the tool name yet. If it's available, send - # it. otherwise, wait until it's available. - if not self.current_tool_name_sent: - if current_tool_call is None: - return None - function_name: str | None = current_tool_call.get("name") - if function_name: - self.current_tool_name_sent = True - return DeltaMessage( - tool_calls=[ - DeltaToolCall( - index=self.current_tool_id, - type="function", - id=make_tool_call_id(), - function=DeltaFunctionCall( - name=function_name - ).model_dump(exclude_none=True), - ) - ] - ) - else: - return None - - # case -- otherwise, send the tool call delta - - # if the tool call portion is None, send the delta as text - if tool_call_portion is None: - # if there's text but not tool calls, send that - - # otherwise None to skip chunk - delta = ( - DeltaMessage(content=delta_text) - if text_portion is not None - else None - ) - return delta - - # now, the nitty-gritty of tool calls - # now we have the portion to parse as tool call. - - logger.debug( - "Trying to parse current tool call with ID %s", self.current_tool_id - ) - - # if we're starting a new tool call, push an empty object in as - # a placeholder for the arguments - if len(self.prev_tool_call_arr) <= self.current_tool_id: - self.prev_tool_call_arr.append({}) - - # main logic for tool parsing here - compare prev. partially-parsed - # JSON to the current partially-parsed JSON - prev_arguments = self.prev_tool_call_arr[self.current_tool_id].get( - "arguments" - ) - cur_arguments = current_tool_call.get("arguments") - - logger.debug("diffing old arguments: %s", prev_arguments) - logger.debug("against new ones: %s", cur_arguments) - - # case -- no arguments have been created yet. skip sending a delta. - if not cur_arguments and not prev_arguments: - logger.debug("Skipping text %s - no arguments", delta_text) - delta = None - - # case -- prev arguments are defined, but non are now. - # probably impossible, but not a fatal error - just keep going - elif not cur_arguments and prev_arguments: - logger.error( - "should be impossible to have arguments reset " - "mid-call. skipping streaming anything." - ) - delta = None - - # case -- we now have the first info about arguments available from - # autocompleting the JSON - elif cur_arguments and not prev_arguments: - delta = DeltaMessage( - tool_calls=[ - DeltaToolCall( - index=self.current_tool_id, - function=DeltaFunctionCall( - arguments=cur_arguments - ).model_dump(exclude_none=True), - ) - ] - ) - self.streamed_args_for_tool[self.current_tool_id] = cur_arguments - - # last case -- we have an update to existing arguments. - elif cur_arguments and prev_arguments: - if ( - isinstance(delta_text, str) - and cur_arguments != prev_arguments - and len(cur_arguments) > len(prev_arguments) - and cur_arguments.startswith(prev_arguments) - ): - delta_arguments = cur_arguments[len(prev_arguments) :] - logger.debug("got diff %s", delta_text) - - delta = DeltaMessage( - tool_calls=[ - DeltaToolCall( - index=self.current_tool_id, - function=DeltaFunctionCall( - arguments=delta_arguments - ).model_dump(exclude_none=True), - ) - ] - ) - self.streamed_args_for_tool[self.current_tool_id] = cur_arguments - else: - delta = None - - # handle saving the state for the current tool into - # the "prev" list for use in diffing for the next iteration - if self.current_tool_id == len(self.prev_tool_call_arr) - 1: - self.prev_tool_call_arr[self.current_tool_id] = current_tool_call - else: - self.prev_tool_call_arr.append(current_tool_call) - - return delta - - except Exception: - logger.exception("Error trying to handle streaming tool call.") - return None # do not stream a delta. skip this token ID. diff --git a/vllm/tool_parsers/deepseekv41_engine_tool_parser.py b/vllm/tool_parsers/deepseekv41_engine_tool_parser.py deleted file mode 100644 index a94883d06a..0000000000 --- a/vllm/tool_parsers/deepseekv41_engine_tool_parser.py +++ /dev/null @@ -1,8 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from vllm.parser.engine.registered_adapters import DeepSeekV41ParserToolAdapter - - -class DeepSeekV41EngineToolParser(DeepSeekV41ParserToolAdapter): # type: ignore[valid-type, misc] - structural_tag_model = "deepseek_v4_1" diff --git a/vllm/tool_parsers/deepseekv4_engine_tool_parser.py b/vllm/tool_parsers/deepseekv4_engine_tool_parser.py deleted file mode 100644 index 7e3ebf2691..0000000000 --- a/vllm/tool_parsers/deepseekv4_engine_tool_parser.py +++ /dev/null @@ -1,8 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from vllm.parser.engine.registered_adapters import DeepSeekV4ParserToolAdapter - - -class DeepSeekV4EngineToolParser(DeepSeekV4ParserToolAdapter): # type: ignore[valid-type, misc] - structural_tag_model = "deepseek_v4" diff --git a/vllm/tool_parsers/dots_tool_parser.py b/vllm/tool_parsers/dots_tool_parser.py deleted file mode 100644 index 311e545365..0000000000 --- a/vllm/tool_parsers/dots_tool_parser.py +++ /dev/null @@ -1,434 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -import json -from collections.abc import Sequence -from typing import Any - -import regex as re - -from vllm.entrypoints.generate.base.protocol import ( - DeltaFunctionCall, - DeltaMessage, - DeltaToolCall, - ExtractedToolCallInformation, - FunctionCall, - ToolCall, -) -from vllm.entrypoints.openai.chat_completion.protocol import ( - ChatCompletionRequest, - ChatCompletionToolsParam, -) -from vllm.logger import init_logger -from vllm.renderers.chat_utils import make_tool_call_id -from vllm.tokenizers import TokenizerLike -from vllm.tool_parsers.abstract_tool_parser import Tool, ToolParser -from vllm.tool_parsers.utils import is_complete_json, partial_tag_overlap - -logger = init_logger(__name__) - - -class DotsToolParser(ToolParser): - """Parse Dots tool calls in their XML wrapper format. - - The canonical body contains one or more ``invoke`` elements:: - - - - weather in Shanghai - - - - A JSON object with ``name`` and ``arguments`` is also accepted as a - fallback. Multiple wrapper blocks and multiple invokes per block are - supported. - """ - - supports_required_and_named = False - - tool_call_start_token = "" - tool_call_end_token = "" - - _block_regex = re.compile( - rf"{re.escape(tool_call_start_token)}\s*(.*?)\s*" - rf"{re.escape(tool_call_end_token)}", - re.DOTALL, - ) - _invoke_regex = re.compile( - r"[^>]+)>(?P.*?)", - re.DOTALL, - ) - _parameter_regex = re.compile( - r"[^>]+)>(?P.*?)", - re.DOTALL, - ) - - def __init__( - self, - tokenizer: TokenizerLike, - tools: list[Tool] | None = None, - ) -> None: - super().__init__(tokenizer, tools) - self._buffer = "" - - @staticmethod - def _extract_name(value: str) -> str: - value = value.strip() - if len(value) >= 2 and value[0] == value[-1] and value[0] in {'"', "'"}: - return value[1:-1] - return value - - @staticmethod - def _convert_param_value(value: str, param_type: Any) -> Any: - if value.lower() == "null": - return None - - if isinstance(param_type, list): - param_type = next((item for item in param_type if item != "null"), "string") - if not isinstance(param_type, str): - param_type = str(param_type) - param_type = param_type.lower() - - if param_type in {"string", "str", "text"}: - return value - if param_type in {"integer", "int"}: - try: - return int(value) - except (TypeError, ValueError): - return value - if param_type in {"number", "float"}: - try: - number = float(value) - return int(number) if number.is_integer() else number - except (TypeError, ValueError): - return value - if param_type in {"boolean", "bool"}: - return value.lower() in {"true", "1"} - - try: - return json.loads(value) - except (json.JSONDecodeError, TypeError, ValueError): - return value - - def _resolve_param_type( - self, - schema: Any, - defs: dict[str, Any], - depth: int = 0, - ) -> Any | None: - if not isinstance(schema, dict) or depth > 10: - return None - if "type" in schema: - return schema["type"] - - ref = schema.get("$ref") - if isinstance(ref, str) and ref.startswith("#/$defs/"): - return self._resolve_param_type( - defs.get(ref.rsplit("/", 1)[-1]), defs, depth + 1 - ) - - for keyword in ("anyOf", "oneOf", "allOf"): - alternatives = schema.get(keyword) - if not isinstance(alternatives, list): - continue - for alternative in alternatives: - if isinstance(alternative, dict) and alternative.get("type") == "null": - continue - resolved = self._resolve_param_type(alternative, defs, depth + 1) - if resolved is not None: - return resolved - return None - - @staticmethod - def _tool_schema( - name: str, - tools: list[ChatCompletionToolsParam] | None, - ) -> tuple[dict[str, Any], dict[str, Any]]: - for tool in tools or []: - if tool.function.name != name: - continue - schema = tool.function.parameters - if not isinstance(schema, dict): - break - properties = schema.get("properties", {}) - defs = schema.get("$defs", {}) - return ( - properties if isinstance(properties, dict) else {}, - defs if isinstance(defs, dict) else {}, - ) - return {}, {} - - def _parse_xml_invoke( - self, - match: re.Match[str], - tools: list[ChatCompletionToolsParam] | None, - ) -> dict[str, Any]: - name = self._extract_name(match.group("name")) - properties, defs = self._tool_schema(name, tools) - arguments: dict[str, Any] = {} - - for parameter in self._parameter_regex.finditer(match.group("body")): - param_name = self._extract_name(parameter.group("name")) - value = parameter.group("value").strip() - param_type: Any = "string" - if param_name in properties: - param_type = ( - self._resolve_param_type(properties[param_name], defs) or "string" - ) - arguments[param_name] = self._convert_param_value(value, param_type) - - return {"name": name, "arguments": arguments} - - def _parse_block( - self, - content: str, - tools: list[ChatCompletionToolsParam] | None, - ) -> list[dict[str, Any]]: - content = content.strip() - if content.startswith(" set[str]: - return {tool.function.name for tool in tools or []} - - def _validated_call( - self, - parsed: dict[str, Any], - tools: list[ChatCompletionToolsParam] | None, - ) -> tuple[str, dict[str, Any]] | None: - name = parsed.get("name") - if not isinstance(name, str) or name not in self._known_tool_names(tools): - return None - arguments = parsed.get("arguments", parsed.get("parameters", {})) or {} - if not isinstance(arguments, dict): - return None - return name, arguments - - def extract_tool_calls( - self, - model_output: str, - request: ChatCompletionRequest, - ) -> ExtractedToolCallInformation: - marker_index = model_output.find(self.tool_call_start_token) - if marker_index == -1: - return ExtractedToolCallInformation( - tools_called=False, - tool_calls=[], - content=model_output, - ) - - tool_calls: list[ToolCall] = [] - for block in self._block_regex.finditer(model_output): - try: - for parsed in self._parse_block(block.group(1), request.tools): - validated = self._validated_call(parsed, request.tools) - if validated is None: - continue - name, arguments = validated - tool_calls.append( - ToolCall( - function=FunctionCall( - name=name, - arguments=json.dumps(arguments, ensure_ascii=False), - ) - ) - ) - except (json.JSONDecodeError, TypeError, ValueError) as exc: - logger.warning("Failed to parse Dots tool call: %s", exc) - - normal_text = model_output[:marker_index].strip() - if not tool_calls: - return ExtractedToolCallInformation( - tools_called=False, - tool_calls=[], - content=model_output, - ) - return ExtractedToolCallInformation( - tools_called=True, - tool_calls=tool_calls, - content=normal_text or None, - ) - - def _append_complete_stream_call( - self, - name: str, - arguments: dict[str, Any], - tool_calls: list[DeltaToolCall], - ) -> None: - self.current_tool_id += 1 - serialized = json.dumps(arguments, ensure_ascii=False) - self.prev_tool_call_arr.append({"name": name, "arguments": arguments}) - self.streamed_args_for_tool.append(serialized) - tool_calls.append( - DeltaToolCall( - index=self.current_tool_id, - id=make_tool_call_id(), - type="function", - function=DeltaFunctionCall(name=name, arguments=serialized), - ) - ) - - def _stream_complete_json_body( - self, - tools: list[ChatCompletionToolsParam] | None, - tool_calls: list[DeltaToolCall], - ) -> None: - content = self._buffer[len(self.tool_call_start_token) :].strip() - if not content or not is_complete_json(content): - return - - try: - parsed = json.loads(content) - except (json.JSONDecodeError, TypeError, ValueError): - return - if not isinstance(parsed, dict): - return - validated = self._validated_call(parsed, tools) - if validated is None: - return - - name, arguments = validated - serialized = json.dumps(arguments, ensure_ascii=False) - if not self.current_tool_name_sent: - self.current_tool_id += 1 - tool_calls.append( - DeltaToolCall( - index=self.current_tool_id, - id=make_tool_call_id(), - type="function", - function=DeltaFunctionCall(name=name, arguments=""), - ) - ) - self.prev_tool_call_arr.append({"name": name, "arguments": arguments}) - self.streamed_args_for_tool.append("") - self.current_tool_name_sent = True - - streamed = self.streamed_args_for_tool[self.current_tool_id] - if serialized.startswith(streamed): - argument_diff = serialized[len(streamed) :] - if argument_diff: - tool_calls.append( - DeltaToolCall( - index=self.current_tool_id, - function=DeltaFunctionCall(arguments=argument_diff), - ) - ) - self.streamed_args_for_tool[self.current_tool_id] += argument_diff - - def extract_tool_calls_streaming( - self, - previous_text: str, - current_text: str, - delta_text: str, - previous_token_ids: Sequence[int], - current_token_ids: Sequence[int], - delta_token_ids: Sequence[int], - request: ChatCompletionRequest, - ) -> DeltaMessage | None: - del current_text, previous_token_ids, current_token_ids, delta_token_ids - if not previous_text: - self._buffer = "" - self.prev_tool_call_arr = [] - self.current_tool_id = -1 - self.current_tool_name_sent = False - self.streamed_args_for_tool = [] - - self._buffer += delta_text - normal_parts: list[str] = [] - tool_calls: list[DeltaToolCall] = [] - - while self._buffer: - marker_index = self._buffer.find(self.tool_call_start_token) - if marker_index == -1: - partial_len = partial_tag_overlap( - self._buffer, self.tool_call_start_token - ) - if partial_len: - normal_parts.append(self._buffer[:-partial_len]) - self._buffer = self._buffer[-partial_len:] - else: - normal_parts.append(self._buffer) - self._buffer = "" - normal_parts = [ - part.replace(self.tool_call_end_token, "") for part in normal_parts - ] - break - - if marker_index > 0: - normal_parts.append(self._buffer[:marker_index]) - self._buffer = self._buffer[marker_index:] - - end_index = self._buffer.find( - self.tool_call_end_token, len(self.tool_call_start_token) - ) - if end_index == -1: - self._stream_complete_json_body(request.tools, tool_calls) - break - - content = self._buffer[len(self.tool_call_start_token) : end_index] - self._buffer = self._buffer[end_index + len(self.tool_call_end_token) :] - try: - parsed_calls = self._parse_block(content, request.tools) - if not parsed_calls: - raise ValueError("Dots tool-call block contains no invoke") - - block_had_streamed_call = self.current_tool_name_sent - valid_calls = [ - validated - for parsed in parsed_calls - if (validated := self._validated_call(parsed, request.tools)) - is not None - ] - if self.current_tool_name_sent and valid_calls: - name, arguments = valid_calls.pop(0) - serialized = json.dumps(arguments, ensure_ascii=False) - streamed = self.streamed_args_for_tool[self.current_tool_id] - if serialized.startswith(streamed): - remaining = serialized[len(streamed) :] - if remaining: - tool_calls.append( - DeltaToolCall( - index=self.current_tool_id, - function=DeltaFunctionCall(arguments=remaining), - ) - ) - self.prev_tool_call_arr[self.current_tool_id] = { - "name": name, - "arguments": arguments, - } - self.streamed_args_for_tool[self.current_tool_id] = serialized - - for name, arguments in valid_calls: - self._append_complete_stream_call(name, arguments, tool_calls) - - if not valid_calls and not block_had_streamed_call: - normal_parts.append(content.strip()) - except (json.JSONDecodeError, TypeError, ValueError) as exc: - logger.warning("Failed to parse streamed Dots tool call: %s", exc) - normal_parts.append(content.strip()) - - self.current_tool_name_sent = False - - content_delta = "".join(normal_parts) - if not content_delta and not tool_calls: - return None - return DeltaMessage(content=content_delta or None, tool_calls=tool_calls) - - def flush_pending_normal_text(self) -> str: - """Return a partial opening marker as text when generation ends.""" - if not self._buffer or self.tool_call_start_token in self._buffer: - return "" - normal_text = self._buffer.replace(self.tool_call_end_token, "") - self._buffer = "" - return normal_text diff --git a/vllm/tool_parsers/ernie45_tool_parser.py b/vllm/tool_parsers/ernie45_tool_parser.py deleted file mode 100644 index 3c654b4489..0000000000 --- a/vllm/tool_parsers/ernie45_tool_parser.py +++ /dev/null @@ -1,211 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -import json -from collections.abc import Sequence - -import regex as re - -from vllm.entrypoints.generate.base.protocol import ( - DeltaFunctionCall, - DeltaMessage, - DeltaToolCall, - ExtractedToolCallInformation, - FunctionCall, - ToolCall, -) -from vllm.entrypoints.openai.chat_completion.protocol import ( - ChatCompletionRequest, -) -from vllm.logger import init_logger -from vllm.tokenizers import TokenizerLike -from vllm.tool_parsers.abstract_tool_parser import ( - Tool, - ToolParser, -) - -logger = init_logger(__name__) - - -class Ernie45ToolParser(ToolParser): - def __init__(self, tokenizer: TokenizerLike, tools: list[Tool] | None = None): - """Ernie thinking model format: - abc\n\n\n\n\ndef\n\n - """ - super().__init__(tokenizer, tools) - self.prev_tool_call_arr: list[dict] = [] - self.current_tool_id = -1 - self.streamed_args_for_tool: list[str] = [] - self.think_end_token = "" - self.response_start_token: str = "" - self.response_end_token: str = "" - self.tool_call_start_token = "" - self.tool_call_end_token = "" - self.tool_calls_start_token = self.tool_call_start_token - self.newline_token: str = "<0x0A>" - - self.tool_call_regex = re.compile( - r"\s*(?P\{.*?\})\s*", re.DOTALL - ) - - if not self.model_tokenizer: - raise ValueError( - "The model tokenizer must be passed to the ToolParser " - "constructor during construction." - ) - - self.think_end_token_id = self.vocab.get(self.think_end_token) - self.response_start_token_id = self.vocab.get(self.response_start_token) - self.response_end_token_id = self.vocab.get(self.response_end_token) - self.tool_call_start_token_id = self.vocab.get(self.tool_call_start_token) - self.tool_call_end_token_id = self.vocab.get(self.tool_call_end_token) - self.newline_token_id = self.vocab.get(self.newline_token) - self.parser_token_ids = [ - self.think_end_token_id, - self.response_start_token_id, - self.response_end_token_id, - ] - - self._buffer = "" - - def extract_tool_calls( - self, - model_output: str, - request: ChatCompletionRequest, - ) -> ExtractedToolCallInformation: - # sanity check; avoid unnecessary processing - if self.tool_calls_start_token not in model_output: - return ExtractedToolCallInformation( - tools_called=False, tool_calls=[], content=model_output - ) - - else: - try: - tool_call_json_list = self.tool_call_regex.findall(model_output) - - tool_calls = [] - for tool_call_json in tool_call_json_list: - tool_call_dict = json.loads(tool_call_json) - args_str = json.dumps( - tool_call_dict.get("arguments", {}), ensure_ascii=False - ) - tool_calls.append( - ToolCall( - type="function", - function=FunctionCall( - name=tool_call_dict.get("name", ""), - arguments=args_str, - ), - ) - ) - - content = model_output[ - : model_output.find(self.tool_calls_start_token) - ].rstrip("\n") - return ExtractedToolCallInformation( - tools_called=True, - tool_calls=tool_calls, - content=content if content else None, - ) - - except Exception: - logger.exception("Error in extracting tool call from response.") - return ExtractedToolCallInformation( - tools_called=False, tool_calls=[], content=model_output - ) - - def extract_tool_calls_streaming( - self, - previous_text: str, - current_text: str, - delta_text: str, - previous_token_ids: Sequence[int], - current_token_ids: Sequence[int], - delta_token_ids: Sequence[int], - request: ChatCompletionRequest, - ) -> DeltaMessage | None: - self._buffer += delta_text - cur_text = self._buffer - start_idx = cur_text.find(self.tool_call_start_token) - if start_idx == -1: - self._buffer = "" - # At least one toolcall has been completed - if self.current_tool_id > 0: - cur_text = "" - if self.current_tool_id == -1 and all( - token_id == self.newline_token_id for token_id in previous_token_ids - ): - cur_text = cur_text.strip("\n") - - # handle when tool_call is not triggered - # cur_text === delta_text - content = cur_text - if self.response_start_token_id in delta_token_ids: - content = content.lstrip("\n") - response_start_idx = content.find(self.response_start_token) - content = content[response_start_idx + len(self.response_start_token) :] - # if have , remove it - response_end_idx = content.rfind(self.response_end_token) - if response_end_idx != -1: - content = content[:response_end_idx] - elif self.response_end_token_id in delta_token_ids: - response_end_idx = content.rfind(self.response_end_token) - content = content[:response_end_idx] - # remove \n after or or - if ( - len(previous_token_ids) > 0 - and previous_token_ids[-1] in self.parser_token_ids - ) and ( - len(delta_token_ids) > 0 and delta_token_ids[0] == self.newline_token_id - ): - content = content.lstrip("\n") - - return DeltaMessage(content=content if content else None) - logger.debug("cur_text = %s", cur_text) - end_idx = cur_text.find(self.tool_call_end_token) - if end_idx != -1: - if self.current_tool_id == -1: - self.current_tool_id = 0 - self.prev_tool_call_arr = [] - self.streamed_args_for_tool = [] - while len(self.prev_tool_call_arr) <= self.current_tool_id: - self.prev_tool_call_arr.append({}) - while len(self.streamed_args_for_tool) <= self.current_tool_id: - self.streamed_args_for_tool.append("") - - extracted_tool_calls = self.extract_tool_calls( - cur_text[: end_idx + len(self.tool_call_end_token)], request - ) - - if len(extracted_tool_calls.tool_calls) == 0: - logger.warning("Failed to extract any tool calls.") - return None - tool_call = extracted_tool_calls.tool_calls[0] - self.prev_tool_call_arr[self.current_tool_id] = { - "name": tool_call.function.name, - "arguments": json.loads(tool_call.function.arguments), - } - self.streamed_args_for_tool[self.current_tool_id] = ( - tool_call.function.arguments - ) - delta = DeltaMessage( - content=extracted_tool_calls.content, - tool_calls=[ - DeltaToolCall( - index=self.current_tool_id, - id=tool_call.id, - type=tool_call.type, - function=DeltaFunctionCall( - name=tool_call.function.name, - arguments=tool_call.function.arguments, - ), - ) - ], - ) - self.current_tool_id += 1 - self._buffer = cur_text[end_idx + len(self.tool_call_end_token) :] - return delta - - self._buffer = cur_text[start_idx:] - content = cur_text[:start_idx].rstrip("\n") - return DeltaMessage(content=content if content else None) diff --git a/vllm/tool_parsers/functiongemma_tool_parser.py b/vllm/tool_parsers/functiongemma_tool_parser.py deleted file mode 100644 index 6fcae6e552..0000000000 --- a/vllm/tool_parsers/functiongemma_tool_parser.py +++ /dev/null @@ -1,313 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -import json -from collections.abc import Sequence - -import regex as re - -from vllm.entrypoints.generate.base.protocol import ( - DeltaFunctionCall, - DeltaMessage, - DeltaToolCall, - ExtractedToolCallInformation, - FunctionCall, - ToolCall, -) -from vllm.entrypoints.openai.chat_completion.protocol import ( - ChatCompletionRequest, -) -from vllm.entrypoints.openai.responses.protocol import ResponsesRequest -from vllm.logger import init_logger -from vllm.renderers.chat_utils import make_tool_call_id -from vllm.tokenizers import TokenizerLike -from vllm.tool_parsers.abstract_tool_parser import Tool, ToolParser - -logger = init_logger(__name__) - - -class FunctionGemmaToolParser(ToolParser): - """Tool parser for Google's FunctionGemma model (google/functiongemma-270m-it). - - Handles the FunctionGemma function call format: - call:func_name{param:value} - """ - - # FunctionGemma tokens - tool_call_start_token: str = "" - tool_call_end_token: str = "" - - # Regex patterns - tool_call_regex: re.Pattern = re.compile( - r"call:(\w+)\{(.*?)\}" - r"|call:(\w+)\{(.*)", - re.DOTALL, - ) - arg_regex: re.Pattern = re.compile( - r"(\w+):(.*?)", - re.DOTALL, - ) - - def __init__(self, tokenizer: TokenizerLike, tools: list[Tool] | None = None): - super().__init__(tokenizer, tools) - - # Streaming state - self.current_tool_name_sent: bool = False - self.prev_tool_call_arr: list[dict] = [] - self.current_tool_id: int = -1 - self.streamed_args_for_tool: list[str] = [] - self.buffered_delta_text = "" - - def _parse_arguments(self, args_str: str) -> dict: - """Parse FunctionGemma argument string into a dictionary.""" - arguments: dict = {} - if not args_str: - return arguments - - matches = self.arg_regex.findall(args_str) - for key, value in matches: - try: - parsed_value = json.loads(value) - arguments[key] = parsed_value - except json.JSONDecodeError: - arguments[key] = value - - return arguments - - def adjust_request( - self, request: ChatCompletionRequest | ResponsesRequest - ) -> ChatCompletionRequest | ResponsesRequest: - request = super().adjust_request(request) - if request.tools and request.tool_choice != "none": - request.skip_special_tokens = False - return request - - def extract_tool_calls( - self, - model_output: str, - request: ChatCompletionRequest, - ) -> ExtractedToolCallInformation: - if self.tool_call_start_token not in model_output: - return ExtractedToolCallInformation( - tools_called=False, tool_calls=[], content=model_output - ) - - try: - matches = self.tool_call_regex.findall(model_output) - - if not matches: - return ExtractedToolCallInformation( - tools_called=False, tool_calls=[], content=model_output - ) - - tool_calls: list[ToolCall] = [] - - for match in matches: - func_name = match[0] if match[0] else match[2] - args_str = match[1] if match[1] else match[3] - - if not func_name: - continue - - arguments = self._parse_arguments(args_str) - - tool_calls.append( - ToolCall( - type="function", - function=FunctionCall( - name=func_name, - arguments=json.dumps(arguments, ensure_ascii=False), - ), - ) - ) - - if tool_calls: - content_end = model_output.find(self.tool_call_start_token) - content = ( - model_output[:content_end].strip() if content_end > 0 else None - ) - - return ExtractedToolCallInformation( - tools_called=True, - tool_calls=tool_calls, - content=content if content else None, - ) - - return ExtractedToolCallInformation( - tools_called=False, tool_calls=[], content=model_output - ) - - except Exception: - logger.exception("Error extracting tool calls from FunctionGemma response") - return ExtractedToolCallInformation( - tools_called=False, tool_calls=[], content=model_output - ) - - def _buffer_delta_text(self, delta_text: str) -> str: - """Buffer incoming delta text to handle multi-token special sequences.""" - potential_start = "" - potential_end = "" - - combined = self.buffered_delta_text + delta_text - - if combined.endswith(potential_start) or combined.endswith(potential_end): - self.buffered_delta_text = "" - return combined - - for tag in [potential_start, potential_end]: - for i in range(1, len(tag)): - if combined.endswith(tag[:i]): - self.buffered_delta_text = combined[-(i):] - return combined[:-i] - - self.buffered_delta_text = "" - return combined - - def extract_tool_calls_streaming( - self, - previous_text: str, - current_text: str, - delta_text: str, - previous_token_ids: Sequence[int], - current_token_ids: Sequence[int], - delta_token_ids: Sequence[int], - request: ChatCompletionRequest, - ) -> DeltaMessage | None: - delta_text = self._buffer_delta_text(delta_text) - current_text = previous_text + delta_text - - if self.tool_call_start_token not in current_text: - if delta_text: - return DeltaMessage(content=delta_text) - return None - - try: - start_count = current_text.count(self.tool_call_start_token) - end_count = current_text.count(self.tool_call_end_token) - prev_start_count = previous_text.count(self.tool_call_start_token) - prev_end_count = previous_text.count(self.tool_call_end_token) - - if self.tool_call_start_token not in current_text: - return DeltaMessage(content=delta_text) - - # Starting a new function call - if start_count > prev_start_count and start_count > end_count: - self.current_tool_id += 1 - self.current_tool_name_sent = False - self.streamed_args_for_tool.append("") - self.prev_tool_call_arr.append({}) - logger.debug("Starting new tool call %d", self.current_tool_id) - return None - - # In the middle of a function call - if start_count > end_count: - last_start = current_text.rfind(self.tool_call_start_token) - partial_call = current_text[ - last_start + len(self.tool_call_start_token) : - ] - - if partial_call.startswith("call:"): - func_part = partial_call[5:] - - if "{" in func_part: - func_name = func_part.split("{")[0] - args_part = ( - func_part.split("{", 1)[1] if "{" in func_part else "" - ) - - if not self.current_tool_name_sent and func_name: - self.current_tool_name_sent = True - self.prev_tool_call_arr[self.current_tool_id] = { - "name": func_name, - "arguments": {}, - } - return DeltaMessage( - tool_calls=[ - DeltaToolCall( - index=self.current_tool_id, - type="function", - id=make_tool_call_id(), - function=DeltaFunctionCall( - name=func_name - ).model_dump(exclude_none=True), - ) - ] - ) - - if self.current_tool_name_sent and args_part: - current_args = self._parse_arguments(args_part) - if current_args: - current_args_json = json.dumps( - current_args, ensure_ascii=False - ) - prev_streamed = self.streamed_args_for_tool[ - self.current_tool_id - ] - - if len(current_args_json) > len(prev_streamed): - diff = current_args_json[len(prev_streamed) :] - self.streamed_args_for_tool[ - self.current_tool_id - ] = current_args_json - self.prev_tool_call_arr[self.current_tool_id][ - "arguments" - ] = current_args - - return DeltaMessage( - tool_calls=[ - DeltaToolCall( - index=self.current_tool_id, - function=DeltaFunctionCall( - arguments=diff - ).model_dump(exclude_none=True), - ) - ] - ) - - return None - - # Function call just ended - if end_count > prev_end_count: - if self.current_tool_id >= 0 and self.current_tool_id < len( - self.prev_tool_call_arr - ): - all_calls = self.tool_call_regex.findall(current_text) - args = {} - if self.current_tool_id < len(all_calls): - match = all_calls[self.current_tool_id] - if match[0]: - args_str = match[1] - args = self._parse_arguments(args_str) - self.prev_tool_call_arr[self.current_tool_id][ - "arguments" - ] = args - - if args: - args_json = json.dumps(args, ensure_ascii=False) - prev_streamed = self.streamed_args_for_tool[ - self.current_tool_id - ] - if len(args_json) > len(prev_streamed): - diff = args_json[len(prev_streamed) :] - self.streamed_args_for_tool[self.current_tool_id] = ( - args_json - ) - return DeltaMessage( - tool_calls=[ - DeltaToolCall( - index=self.current_tool_id, - function=DeltaFunctionCall( - arguments=diff - ).model_dump(exclude_none=True), - ) - ] - ) - return None - - if delta_text: - return DeltaMessage(content=delta_text) - return None - - except Exception: - logger.exception("Error in streaming tool call extraction") - return None diff --git a/vllm/tool_parsers/gemma4_engine_tool_parser.py b/vllm/tool_parsers/gemma4_engine_tool_parser.py deleted file mode 100644 index fcb578b129..0000000000 --- a/vllm/tool_parsers/gemma4_engine_tool_parser.py +++ /dev/null @@ -1,41 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from openai.types.responses import ToolChoiceCustom, ToolChoiceFunction - -from vllm.entrypoints.openai.chat_completion.protocol import ( - ChatCompletionNamedToolChoiceParam, - ChatCompletionRequest, -) -from vllm.entrypoints.openai.responses.protocol import ResponsesRequest -from vllm.parser.engine.registered_adapters import Gemma4ParserToolAdapter - - -class Gemma4EngineToolParser(Gemma4ParserToolAdapter): # type: ignore[valid-type, misc] - supports_required_and_named = False - - def adjust_request( - self, request: ChatCompletionRequest | ResponsesRequest - ) -> ChatCompletionRequest | ResponsesRequest: - """Skip structured-output JSON for required/named tool choice. - - Gemma4 emits its native ``<|tool_call>call:...`` syntax, which the - parser extracts directly. The base ``ToolParser.adjust_request`` would - set ``structured_outputs`` for required/named and force JSON via guided - decoding, conflicting with that native syntax (it leaks as content and - crashes EngineCore under speculative decoding). Skip it so the model - emits its native format (mirrors the GLM4 parser). - """ - if request.tools: - tc = request.tool_choice - if tc == "required" or isinstance( - tc, - ( - ChatCompletionNamedToolChoiceParam, - ToolChoiceFunction, - ToolChoiceCustom, - ), - ): - request.skip_special_tokens = False - return request - return super().adjust_request(request) diff --git a/vllm/tool_parsers/gemma4_utils.py b/vllm/tool_parsers/gemma4_utils.py deleted file mode 100644 index 33ad20b193..0000000000 --- a/vllm/tool_parsers/gemma4_utils.py +++ /dev/null @@ -1,158 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -# Copyright 2025 Google Inc. HuggingFace Inc. team. All rights reserved. - -"""Gemma4 tool call parsing utilities for offline inference. - -Standalone functions that parse decoded model text to extract tool calls -from Gemma4 models. These are pure-Python utilities with zero heavy -dependencies — they work on raw decoded strings from any inference -backend (vLLM, HuggingFace, TGI, etc.). - -For the OpenAI-compatible API server tool parser (streaming + -non-streaming), see ``vllm.tool_parsers.gemma4_tool_parser``. -For thinking/reasoning output parsing, see -``vllm.reasoning.gemma4_utils``. - -Usage with vLLM offline inference:: - - from vllm import LLM, SamplingParams - from vllm.tool_parsers.gemma4_utils import ( - parse_tool_calls, - has_tool_response_tag, - ) - - llm = LLM(model="google/gemma-4-it") - outputs = llm.generate(prompt, SamplingParams(...)) - text = tokenizer.decode(outputs[0].outputs[0].token_ids, skip_special_tokens=False) - - # Extract tool calls - tool_calls = parse_tool_calls(text) - for tc in tool_calls: - print(f"{tc['name']}({tc['arguments']})") - -Ported from ``transformers.models.gemma4.utils_gemma4`` so that vLLM users -do not need a transformers dependency for output parsing. -""" - -import regex as re - -_TOOL_RESPONSE_START_TAG = "<|tool_response>" - - -def _parse_tool_arguments(args_str: str) -> dict[str, str]: - """Parse tool call arguments from the Gemma4 compact format. - - Delegates to the native ``<|"|>``-aware parser from - ``vllm.parser.gemma4``, which handles internal quotes, nested - objects, arrays, and all Gemma4 value types correctly. - - Args: - args_str: Raw argument string from inside ``call:name{...}``. - - Returns: - Dictionary of argument name → string value. - - """ - if not args_str or not args_str.strip(): - return {} - - from vllm.parser.gemma4 import _parse_gemma4_args - - parsed = _parse_gemma4_args(args_str) - return {k: str(v) if not isinstance(v, str) else v for k, v in parsed.items()} - - -def parse_tool_calls(text: str, *, strict: bool = False) -> list[dict]: - """Parse tool calls from decoded Gemma4 model output. - - Uses a tiered parsing strategy to handle known output variations in - Gemma4 models, which may emit - non-standard tool call formats. - - Parsing tiers: - 1. **Standard**: ``<|tool_call>call:name{args}`` - (special token IDs 48/49 in decoded text) - 2. **Fallback** (when ``strict=False``): bare ``call:name{args}`` - patterns, including ``name{args}`` (fragmented tokens from - multimodal inputs) - - Args: - text: Decoded model output text (from ``tokenizer.decode(..., - skip_special_tokens=False)``). - strict: If ``True``, only match the standard ``<|tool_call>`` format. - If ``False`` (default), also try fallback patterns for - known Gemma4 output variations. - - Returns: - A list of dicts, each with keys: - - ``"name"``: The tool function name (e.g. ``"get_weather"``). - - ``"arguments"``: A dict of argument name → value. - - Example:: - - >>> from vllm.tool_parsers.gemma4_utils import parse_tool_calls - >>> output = tokenizer.decode(outputs[0], skip_special_tokens=False) - >>> tool_calls = parse_tool_calls(output) - >>> for tc in tool_calls: - ... print(f"Call: {tc['name']}({tc['arguments']})") - - """ - results = [] - - # Tier 1: Standard format with special tokens. - # <|tool_call>call:name{args} - # Note: Some Gemma4 models emit instead of . - standard_pattern = r"<\|tool_call\>call:(\w+)\{(.*?)\}(?:|)" - for match in re.finditer(standard_pattern, text, re.DOTALL): - name, args_str = match.group(1), match.group(2) - results.append( - { - "name": name, - "arguments": _parse_tool_arguments(args_str), - } - ) - - if results or strict: - return results - - # Tier 2: Fallback for known Gemma4 output variations. - # Matches: name{args}, call:name{args}, or bare call:name{args} - fallback_pattern = r"(?:|(?:^|\s)call:)(\w+)\{(.*?)\}" - for match in re.finditer(fallback_pattern, text, re.DOTALL): - name, args_str = match.group(1), match.group(2) - results.append( - { - "name": name, - "arguments": _parse_tool_arguments(args_str), - } - ) - - return results - - -def has_tool_response_tag(text: str) -> bool: - """Check if model output properly ends with a tool response tag. - - Some Gemma4 models sometimes emit ```` instead of - ``<|tool_response>`` after a tool call. This helper detects - whether the model used the proper termination, so callers can - decide whether to inject ``<|tool_response>`` into the next prompt. - - Args: - text: Decoded model output text. - - Returns: - ``True`` if the output ends with ``<|tool_response>`` - (proper behavior), ``False`` otherwise. - - Example:: - - >>> from vllm.tool_parsers.gemma4_utils import has_tool_response_tag - >>> if not has_tool_response_tag(model_output): - ... # Model used instead — inject <|tool_response> manually - ... next_prompt = "<|tool_response>" + tool_result - - """ - stripped = text.rstrip() - return stripped.endswith(_TOOL_RESPONSE_START_TAG) diff --git a/vllm/tool_parsers/gigachat3_tool_parser.py b/vllm/tool_parsers/gigachat3_tool_parser.py deleted file mode 100644 index 7f9bdf86d6..0000000000 --- a/vllm/tool_parsers/gigachat3_tool_parser.py +++ /dev/null @@ -1,214 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -import json -from collections.abc import Sequence - -import regex as re - -from vllm.entrypoints.generate.base.protocol import ( - DeltaFunctionCall, - DeltaMessage, - DeltaToolCall, - ExtractedToolCallInformation, - FunctionCall, - ToolCall, -) -from vllm.entrypoints.openai.chat_completion.protocol import ( - ChatCompletionRequest, -) -from vllm.entrypoints.openai.responses.protocol import ResponsesRequest -from vllm.logger import init_logger -from vllm.renderers.chat_utils import make_tool_call_id -from vllm.tokenizers import TokenizerLike -from vllm.tool_parsers.abstract_tool_parser import Tool, ToolParser - -logger = init_logger(__name__) - -REGEX_FUNCTION_CALL = re.compile( - r"(?:function call<\|role_sep\|>\n|<\|function_call\|>)(.*)", - re.DOTALL, -) - -REGEX_CONTENT_PATTERN = re.compile( - r"^(.*?)(?:<\|message_sep\|>|<\|function_call\|>)", - re.DOTALL, -) - -NAME_REGEX = re.compile( - r'"name"\s*:\s*"([^"]*)"', - re.DOTALL, -) - -ARGS_REGEX = re.compile( - r'"arguments"\s*:\s*(.*)', - re.DOTALL, -) - - -class GigaChat3ToolParser(ToolParser): - def __init__(self, tokenizer: TokenizerLike, tools: list[Tool] | None = None): - super().__init__(tokenizer, tools) - self.tool_started: bool = False - self.tool_name_sent: bool = False - self.tool_id: str | None = None - self.prev_tool_call_arr: list[dict] = [] - self.end_content: bool = False - self.streamed_args_for_tool: list[str] = [] - - def adjust_request( - self, request: ChatCompletionRequest | ResponsesRequest - ) -> ChatCompletionRequest | ResponsesRequest: - request = super().adjust_request(request) - if request.tools and request.tool_choice != "none": - request.skip_special_tokens = False - return request - - def extract_tool_calls( - self, - model_output: str, - request: ChatCompletionRequest, - ) -> ExtractedToolCallInformation: - function_call = None - content = None - if model_output.rstrip().endswith(""): - model_output = model_output[: model_output.rfind("")] - m_func = REGEX_FUNCTION_CALL.search(model_output) - if m_func: - try: - function_call = json.loads(m_func.group(1), strict=False) - if ( - isinstance(function_call, dict) - and "name" in function_call - and "arguments" in function_call - ): - if not isinstance(function_call["arguments"], dict): - function_call = None - else: - function_call = None - except json.JSONDecodeError: - return ExtractedToolCallInformation( - tools_called=False, - tool_calls=[], - content=model_output, - ) - m_content = REGEX_CONTENT_PATTERN.search(model_output) - content = m_content.group(1) if m_content else model_output - if not function_call: - return ExtractedToolCallInformation( - tools_called=False, - tool_calls=[], - content=content if content else None, - ) - name = function_call["name"] - args = function_call["arguments"] - if not isinstance(args, str): - args = json.dumps(function_call["arguments"], ensure_ascii=False) - return ExtractedToolCallInformation( - tools_called=True, - tool_calls=[ - ToolCall( - type="function", - function=FunctionCall( - name=name, - arguments=args, - ), - ) - ], - content=content if content else None, - ) - - def extract_tool_calls_streaming( - self, - previous_text: str, - current_text: str, - delta_text: str, - previous_token_ids: Sequence[int], - current_token_ids: Sequence[int], - delta_token_ids: Sequence[int], - request: ChatCompletionRequest, - ) -> DeltaMessage | None: - content = None - func_name = None - cur_args = None - m_func = REGEX_FUNCTION_CALL.search(current_text) - if not self.tool_started: - m_content = REGEX_CONTENT_PATTERN.search(delta_text) - if m_content: - content = m_content.group(1) - self.end_content = True - else: - if not self.end_content: - content = delta_text - if m_func: - self.tool_started = True - if content: - return DeltaMessage(content=content) - if not m_func: - return None - json_tail = m_func.group(1).strip() - name_match = NAME_REGEX.search(json_tail) - if name_match: - func_name = name_match.group(1) - args_match = ARGS_REGEX.search(json_tail) - if args_match: - cur_args = args_match.group(1).strip() - if cur_args.endswith(""): - cur_args = cur_args[: -len("")] - if cur_args.endswith("}"): # last '}' end of json - try: - candidate = cur_args[:-1].strip() - json.loads(candidate, strict=False) - cur_args = candidate - except json.JSONDecodeError: - pass - if not self.prev_tool_call_arr: - self.prev_tool_call_arr.append({}) - if not self.tool_name_sent: - if not func_name: - return None - self.tool_name_sent = True - self.tool_id = make_tool_call_id() - self.prev_tool_call_arr[0]["name"] = func_name - return DeltaMessage( - tool_calls=[ - DeltaToolCall( - index=0, - id=self.tool_id, - type="function", - function=DeltaFunctionCall( - name=func_name, - ).model_dump(exclude_none=True), - ) - ], - ) - if cur_args is None: - return None - prev_args = self.prev_tool_call_arr[0].get("arguments_str", "") - if not prev_args: - delta_args = cur_args - elif cur_args.startswith(prev_args): - delta_args = cur_args[len(prev_args) :] - else: - return None - if not delta_args: - return None - self.prev_tool_call_arr[0]["arguments_str"] = cur_args - try: - args_dict = json.loads(cur_args, strict=False) - self.prev_tool_call_arr[0]["arguments"] = args_dict - except json.JSONDecodeError: - self.prev_tool_call_arr[0]["arguments"] = {} - if len(self.streamed_args_for_tool) <= 0: - self.streamed_args_for_tool.append("") - self.streamed_args_for_tool[0] = cur_args - return DeltaMessage( - tool_calls=[ - DeltaToolCall( - index=0, - function=DeltaFunctionCall( - arguments=delta_args, - ).model_dump(exclude_none=True), - ) - ], - ) diff --git a/vllm/tool_parsers/glm47_moe_tool_parser.py b/vllm/tool_parsers/glm47_moe_tool_parser.py deleted file mode 100644 index 6beb9185fb..0000000000 --- a/vllm/tool_parsers/glm47_moe_tool_parser.py +++ /dev/null @@ -1,14 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from __future__ import annotations - -from vllm.parser.engine.registered_adapters import Glm47MoeParserToolAdapter -from vllm.tool_parsers.tool_strict_level import ToolStrictLevel - - -class Glm47MoeModelToolParser(Glm47MoeParserToolAdapter): # type: ignore[valid-type, misc] - supports_required_and_named = False - structural_tag_model = "glm_4_7" - # Constrain auto tool calls with the shallow tag by default. - default_tool_strict_level = ToolStrictLevel.FUNCTION diff --git a/vllm/tool_parsers/granite4_tool_parser.py b/vllm/tool_parsers/granite4_tool_parser.py deleted file mode 100644 index aabd455c67..0000000000 --- a/vllm/tool_parsers/granite4_tool_parser.py +++ /dev/null @@ -1,256 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -import json -from collections.abc import Sequence -from typing import Any, Protocol, TypeVar - -import regex as re - -from vllm.entrypoints.generate.base.protocol import ( - DeltaFunctionCall, - DeltaMessage, - DeltaToolCall, - ExtractedToolCallInformation, - FunctionCall, - ToolCall, -) -from vllm.entrypoints.openai.chat_completion.protocol import ( - ChatCompletionRequest, -) -from vllm.entrypoints.openai.responses.protocol import ResponsesRequest -from vllm.logger import init_logger -from vllm.renderers.chat_utils import make_tool_call_id -from vllm.tokenizers import TokenizerLike -from vllm.tool_parsers.abstract_tool_parser import ( - Tool, - ToolParser, -) - -logger = init_logger(__name__) - - -def dump_args(args: None | dict[str, Any] | str) -> str | None: - if args is None or isinstance(args, str): - return args - else: - return json.dumps(args, ensure_ascii=False) - - -class _FunctionCallCtor(Protocol): - def __init__(self, *, name: str, arguments: str | None): ... - - -FuncT = TypeVar("FuncT", bound=_FunctionCallCtor) - - -class Granite4ToolParser(ToolParser): - def __init__(self, tokenizer: TokenizerLike, tools: list[Tool] | None = None): - super().__init__(tokenizer, tools) - - self.prev_tool_call_arr: list[dict] = [] - self.current_tool_id: int = -1 - self.streamed_args_for_tool = list[str]() - - self.look_ahead = "" - self.in_tc = False - - self.tc_start = "" - self.tc_end = "" - self.start_regex = re.compile(self.tc_start) - self.end_regex = re.compile(self.tc_end) - - def adjust_request( - self, request: ChatCompletionRequest | ResponsesRequest - ) -> ChatCompletionRequest | ResponsesRequest: - request = super().adjust_request(request) - if request.tools and request.tool_choice != "none": - # do not skip special tokens because the tool_call tokens are - # marked "special" in some models. Since they are skipped - # prior to the call to the tool parser, it breaks tool calling. - request.skip_special_tokens = False - return request - - def _collect_results( - self, text_segments: list[str], tc_segments: list[str], cls: type[FuncT] - ) -> tuple[str, list[FuncT]]: - tool_calls_json: list[dict[str, Any]] = [ - json.loads(tc_text) for tc_text in tc_segments - ] - tool_calls = [] - for tc in tool_calls_json: - assert isinstance(tc, dict) - self.prev_tool_call_arr.append(tc) - tool_calls.append( - cls( - name=tc["name"], - arguments=dump_args(tc["arguments"]), - ) - ) - return "".join(text_segments), tool_calls - - def extract_tool_calls( - self, - model_output: str, - request: ChatCompletionRequest, - ) -> ExtractedToolCallInformation: - msg = ExtractedToolCallInformation( - tools_called=False, tool_calls=[], content=model_output - ) - try: - delimiters = [("TC_START", self.tc_start), ("TC_END", self.tc_end)] - pattern = "|".join(f"(?P<{name}>{pattern})" for name, pattern in delimiters) - regex = re.compile(pattern) - - text_segments = list[str]() - tc_segments = list[str]() - last_cut_loc = 0 - - for match in regex.finditer(model_output): - match_type = match.lastgroup - if match_type == "TC_START": - assert not self.in_tc, "Two tool call start tokens found in a row" - if preceding_text := model_output[last_cut_loc : match.start()]: - text_segments.append(preceding_text) - self.in_tc = True - elif match_type == "TC_END": - assert self.in_tc, ( - "Tool call end token found without corresponding start token" - ) - tool_text = model_output[last_cut_loc : match.start()] - assert tool_text, ( - "Expected the model to generate text between tool call tokens" - ) - tc_segments.append(tool_text) - self.in_tc = False - else: - raise ValueError("Unexpected match") - last_cut_loc = match.end() - assert not self.in_tc, "The model generated an incomplete tool call" - if final_text := model_output[last_cut_loc:]: - text_segments.append(final_text) - - content, tool_call_funcs = self._collect_results( - text_segments, tc_segments, FunctionCall - ) - tool_calls = [ - ToolCall( - type="function", - function=func, - ) - for func in tool_call_funcs - ] - msg.tools_called = bool(tool_calls) - msg.tool_calls = tool_calls - msg.content = content or None - except Exception: - logger.exception("Error in extracting tool call from response.") - return msg - - def _tool_extraction_step( - self, - delta_text: str, - ) -> tuple[bool, str, str]: - start_token_pos = start_token_end = end_token_pos = end_token_end = -1 - - if start_match := self.start_regex.search(delta_text, partial=True): - if not start_match.partial: - start_token_pos, start_token_end = start_match.span() - elif start_match.end() > start_match.start(): - start_token_pos = -2 - - if end_match := self.end_regex.search(delta_text): - end_token_pos, end_token_end = end_match.span() - - # Done means that we've exhausted the current buffer - # and need more output from the model - done = True - content = tc_text = "" - - if start_token_pos < 0: - # just streaming text so far - if start_token_pos == -2: - # There is a partial match - content = delta_text[: start_match.start()] - self.look_ahead = delta_text[start_match.start() :] - else: - content = delta_text - - elif not self.in_tc: - # we're entering a new tool call - self.in_tc = True - - content = delta_text[:start_token_pos] - if end_token_pos > 0: - self.start_in_tc = False - tc_text = delta_text[start_token_end:end_token_pos] - self.look_ahead = delta_text[end_token_end:] - done = False # There could be more content already buffered - else: - self.look_ahead = delta_text[start_token_pos:] - - elif end_token_pos < 0: - # we're in between the start and the end token - assert self.in_tc - self.look_ahead = delta_text - else: - # We have found the end - assert self.in_tc - tc_text = delta_text[start_token_end:end_token_pos] - self.in_tc = False - self.look_ahead = delta_text[end_token_end:] - done = False # There could be more content already buffered - return done, content, tc_text - - def extract_tool_calls_streaming( - self, - previous_text: str, - current_text: str, - delta_text: str, - previous_token_ids: Sequence[int], - current_token_ids: Sequence[int], - delta_token_ids: Sequence[int], - request: ChatCompletionRequest, - ) -> DeltaMessage | None: - try: - done = False - text_segments = list[str]() - tc_segments = list[str]() - - while not done: - delta_text = self.look_ahead + delta_text - self.look_ahead = "" - done, content, tc_text = self._tool_extraction_step(delta_text) - if content: - text_segments.append(content) - if tc_text: - tc_segments.append(tc_text) - delta_text = "" - - content, tool_call_funcs = self._collect_results( - text_segments, tc_segments, DeltaFunctionCall - ) - - delta_tool_calls = list[DeltaToolCall]() - for function in tool_call_funcs: - self.current_tool_id += 1 - delta_tool_calls.append( - DeltaToolCall( - id=make_tool_call_id(), - type="function", - index=self.current_tool_id, - function=function.model_dump(exclude_none=True), - ) - ) - self.streamed_args_for_tool.append(function.arguments or "") - - assert self.current_tool_id + 1 == len(self.prev_tool_call_arr) - assert self.current_tool_id + 1 == len(self.streamed_args_for_tool) - - msg = DeltaMessage(content=content or None, tool_calls=delta_tool_calls) - if msg.content or msg.tool_calls: - return msg - - except Exception: - logger.exception("Error trying to handle streaming tool call.") - return None diff --git a/vllm/tool_parsers/granite_20b_fc_tool_parser.py b/vllm/tool_parsers/granite_20b_fc_tool_parser.py deleted file mode 100644 index 8b2ccf58e8..0000000000 --- a/vllm/tool_parsers/granite_20b_fc_tool_parser.py +++ /dev/null @@ -1,275 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -import json -from collections.abc import Sequence -from json import JSONDecoder - -import partial_json_parser -import regex as re -from partial_json_parser.core.options import Allow - -from vllm.entrypoints.generate.base.protocol import ( - DeltaFunctionCall, - DeltaMessage, - DeltaToolCall, - ExtractedToolCallInformation, - FunctionCall, - ToolCall, -) -from vllm.entrypoints.openai.chat_completion.protocol import ( - ChatCompletionRequest, -) -from vllm.logger import init_logger -from vllm.renderers.chat_utils import make_tool_call_id -from vllm.tokenizers import TokenizerLike -from vllm.tool_parsers.abstract_tool_parser import ( - Tool, - ToolParser, -) -from vllm.tool_parsers.utils import ( - consume_space, - find_common_prefix, - is_complete_json, - partial_json_loads, -) - -logger = init_logger(__name__) - - -class Granite20bFCToolParser(ToolParser): - """Tool call parser for the granite-20b-functioncalling model intended - for use with the examples/tool_chat_template_granite20b_fc.jinja - template. - - Used when --enable-auto-tool-choice --tool-call-parser granite-20-fc - are all set - """ - - def __init__(self, tokenizer: TokenizerLike, tools: list[Tool] | None = None): - super().__init__(tokenizer, tools) - - self.bot_token = "" - self.tool_start_token = self.bot_token - self.tool_call_regex = re.compile(r"\s*") - - def extract_tool_calls( - self, model_output: str, request: ChatCompletionRequest - ) -> ExtractedToolCallInformation: - if self.tool_start_token not in model_output: - return ExtractedToolCallInformation( - tools_called=False, tool_calls=[], content=model_output - ) - - dec = JSONDecoder() - try: - matches = list(self.tool_call_regex.finditer(model_output)) - logger.debug("Found %d tool call matches", len(matches)) - - raw_function_calls = [] - - for i, match in enumerate(matches): - # position after the tag - start_of_json = match.end() - # end_index == the start of the next function call - # (if exists) - next_function_call_start = ( - matches[i + 1].start() if i + 1 < len(matches) else None - ) - - raw_function_calls.append( - dec.raw_decode( - model_output[start_of_json:next_function_call_start] - )[0] - ) - - logger.debug("Extracted %d tool calls", len(raw_function_calls)) - tool_calls = [ - ToolCall( - type="function", - function=FunctionCall( - name=function_call["name"], - # function call args are JSON but as a string - arguments=json.dumps( - function_call["arguments"], ensure_ascii=False - ), - ), - ) - for function_call in raw_function_calls - ] - - content = model_output[: model_output.find(self.bot_token)] - return ExtractedToolCallInformation( - tools_called=True, - tool_calls=tool_calls, - content=content if content else None, - ) - - except Exception as e: - logger.error("Error in extracting tool call from response %s", e) - return ExtractedToolCallInformation( - tools_called=False, tool_calls=[], content=model_output - ) - - def extract_tool_calls_streaming( - self, - previous_text: str, - current_text: str, - delta_text: str, - previous_token_ids: Sequence[int], - current_token_ids: Sequence[int], - delta_token_ids: Sequence[int], - request: ChatCompletionRequest, - ) -> DeltaMessage | None: - if len(current_text) < len(self.bot_token) and self.bot_token.startswith( - current_text - ): - return None - - if not current_text.startswith(self.bot_token): - return DeltaMessage(content=delta_text) - - # bit mask flags for partial JSON parsing. If the name hasn't been - # sent yet, don't allow sending - # an incomplete string since OpenAI only ever (as far as I have - # seen) allows sending the entire tool/ function name at once. - flags = Allow.ALL if self.current_tool_name_sent else Allow.ALL & ~Allow.STR - try: - tool_call_arr = [] - is_complete = [] - try: - start_idx = len(self.bot_token) - start_idx = consume_space(start_idx, current_text) - - while start_idx < len(current_text): - (obj, end_idx) = partial_json_loads(current_text[start_idx:], flags) - is_complete.append( - is_complete_json(current_text[start_idx : start_idx + end_idx]) - ) - start_idx += end_idx - start_idx = consume_space(start_idx, current_text) - start_idx += len(self.bot_token) - start_idx = consume_space(start_idx, current_text) - tool_call_arr.append(obj) - except partial_json_parser.core.exceptions.MalformedJSON: - logger.debug("not enough tokens to parse into JSON yet") - return None - - # select as the current tool call the one we're on the state at - current_tool_call: dict = ( - tool_call_arr[self.current_tool_id] if len(tool_call_arr) > 0 else {} - ) - - # case -- if no tokens have been streamed for the tool, e.g. - # only the array brackets, stream nothing - if len(tool_call_arr) == 0: - return None - - # case: we are starting a new tool in the array - # -> array has > 0 length AND length has moved past cursor - elif ( - len(tool_call_arr) > 0 and len(tool_call_arr) > self.current_tool_id + 1 - ): - # if we're moving on to a new call, first make sure we - # haven't missed anything in the previous one that was - # auto-generated due to JSON completions, but wasn't - # streamed to the client yet. - if self.current_tool_id >= 0: - cur_arguments = current_tool_call.get("arguments") - if cur_arguments: - cur_args_json = json.dumps(cur_arguments, ensure_ascii=False) - sent = len(self.streamed_args_for_tool[self.current_tool_id]) - argument_diff = cur_args_json[sent:] - - logger.debug("got arguments diff: %s", argument_diff) - delta = DeltaMessage( - tool_calls=[ - DeltaToolCall( - index=self.current_tool_id, - function=DeltaFunctionCall( - arguments=argument_diff - ).model_dump(exclude_none=True), - ) - ] - ) - self.streamed_args_for_tool[self.current_tool_id] += ( - argument_diff - ) - else: - delta = None - else: - delta = None - # re-set stuff pertaining to progress in the current tool - self.current_tool_id = len(tool_call_arr) - 1 - self.current_tool_name_sent = False - self.streamed_args_for_tool.append("") - logger.debug("starting on new tool %d", self.current_tool_id) - return delta - - # if the current tool name hasn't been sent, send if available - # - otherwise send nothing - elif not self.current_tool_name_sent: - function_name = current_tool_call.get("name") - if function_name: - delta = DeltaMessage( - tool_calls=[ - DeltaToolCall( - index=self.current_tool_id, - type="function", - id=make_tool_call_id(), - function=DeltaFunctionCall( - name=function_name - ).model_dump(exclude_none=True), - ) - ] - ) - self.current_tool_name_sent = True - else: - delta = None - - # now we know we're on the same tool call and we're streaming - # arguments - else: - cur_arguments = current_tool_call.get("arguments") - delta = None - - if cur_arguments: - sent = len(self.streamed_args_for_tool[self.current_tool_id]) - cur_args_json = json.dumps(cur_arguments, ensure_ascii=False) - prev_arguments = self.prev_tool_call_arr[self.current_tool_id].get( - "arguments" - ) - - argument_diff = None - if is_complete[self.current_tool_id]: - argument_diff = cur_args_json[sent:] - elif prev_arguments: - prev_args_json = json.dumps(prev_arguments, ensure_ascii=False) - if cur_args_json != prev_args_json: - prefix = find_common_prefix(prev_args_json, cur_args_json) - argument_diff = prefix[sent:] - - if argument_diff is not None: - delta = DeltaMessage( - tool_calls=[ - DeltaToolCall( - index=self.current_tool_id, - function=DeltaFunctionCall( - arguments=argument_diff - ).model_dump(exclude_none=True), - ) - ] - ) - self.streamed_args_for_tool[self.current_tool_id] += ( - argument_diff - ) - - self.prev_tool_call_arr = tool_call_arr - return delta - - except Exception as e: - logger.error("Error trying to handle streaming tool call: %s", e) - logger.debug( - "Skipping chunk as a result of tool streaming extraction error" - ) - return None diff --git a/vllm/tool_parsers/granite_engine_tool_parser.py b/vllm/tool_parsers/granite_engine_tool_parser.py deleted file mode 100644 index 1fb701d5a4..0000000000 --- a/vllm/tool_parsers/granite_engine_tool_parser.py +++ /dev/null @@ -1,8 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from vllm.parser.engine.registered_adapters import GraniteParserToolAdapter - - -class GraniteEngineToolParser(GraniteParserToolAdapter): # type: ignore[valid-type, misc] - structural_tag_model = None diff --git a/vllm/tool_parsers/hermes_tool_parser.py b/vllm/tool_parsers/hermes_tool_parser.py deleted file mode 100644 index 1fa19d0321..0000000000 --- a/vllm/tool_parsers/hermes_tool_parser.py +++ /dev/null @@ -1,274 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -import json -from collections.abc import Sequence - -import regex as re - -from vllm.entrypoints.generate.base.protocol import ( - DeltaFunctionCall, - DeltaMessage, - DeltaToolCall, - ExtractedToolCallInformation, - FunctionCall, - ToolCall, -) -from vllm.entrypoints.openai.chat_completion.protocol import ( - ChatCompletionRequest, -) -from vllm.entrypoints.openai.responses.protocol import ResponsesRequest -from vllm.logger import init_logger -from vllm.renderers.chat_utils import make_tool_call_id -from vllm.tokenizers import TokenizerLike -from vllm.tool_parsers.abstract_tool_parser import ( - Tool, - ToolParser, -) -from vllm.tool_parsers.utils import is_complete_json, partial_tag_overlap -from vllm.utils.mistral import is_mistral_tokenizer - -logger = init_logger(__name__) - - -class Hermes2ProToolParser(ToolParser): - structural_tag_model = "hermes" - tool_call_start_token: str = "" - tool_call_end_token: str = "" - tool_call_regex = re.compile( - r"(.*?)|(.*)", re.DOTALL - ) - scratch_pad_regex = re.compile(r"(.*?)", re.DOTALL) - - def __init__(self, tokenizer: TokenizerLike, tools: list[Tool] | None = None): - super().__init__(tokenizer, tools) - - if is_mistral_tokenizer(tokenizer): - logger.error("Detected Mistral tokenizer when using a Hermes model") - self.model_tokenizer = tokenizer.tokenizer - - if not self.model_tokenizer: - raise ValueError( - "The model tokenizer must be passed to the ToolParser " - "constructor during construction." - ) - - # Streaming state: what has been sent to the client. - self._sent_content_idx: int = 0 - - def adjust_request( - self, request: ChatCompletionRequest | ResponsesRequest - ) -> ChatCompletionRequest | ResponsesRequest: - request = super().adjust_request(request) - if request.tools and request.tool_choice != "none": - # do not skip special tokens because the tool_call tokens are - # marked "special" in some models. Since they are skipped - # prior to the call to the tool parser, it breaks tool calling. - request.skip_special_tokens = False - return request - - def extract_tool_calls( - self, - model_output: str, - request: ChatCompletionRequest, - ) -> ExtractedToolCallInformation: - # sanity check; avoid unnecessary processing - if self.tool_call_start_token not in model_output: - return ExtractedToolCallInformation( - tools_called=False, tool_calls=[], content=model_output - ) - - else: - try: - # there are two possible captures - between tags, or between a - # tag and end-of-string so the result of - # findall is an array of tuples where one is a function call and - # the other is None - function_call_tuples = self.tool_call_regex.findall(model_output) - - # load the JSON, and then use it to build the Function and - # Tool Call - raw_function_calls = [ - json.loads(match[0] if match[0] else match[1]) - for match in function_call_tuples - ] - tool_calls = [ - ToolCall( - type="function", - function=FunctionCall( - name=function_call["name"], - # function call args are JSON but as a string - arguments=json.dumps( - function_call["arguments"], ensure_ascii=False - ), - ), - ) - for function_call in raw_function_calls - ] - - content = model_output[: model_output.find(self.tool_call_start_token)] - return ExtractedToolCallInformation( - tools_called=True, - tool_calls=tool_calls, - content=content if content else None, - ) - - except Exception: - logger.exception("Error in extracting tool call from response.") - return ExtractedToolCallInformation( - tools_called=False, tool_calls=[], content=model_output - ) - - def _extract_content(self, current_text: str) -> str | None: - """Return unsent non-tool-call text, or None. - - Holds back any suffix that could be a partial tag. - """ - if self.tool_call_start_token not in current_text: - overlap_length = partial_tag_overlap( - current_text, self.tool_call_start_token - ) - sendable_idx = len(current_text) - overlap_length - else: - sendable_idx = current_text.index(self.tool_call_start_token) - - if sendable_idx > self._sent_content_idx: - content = current_text[self._sent_content_idx : sendable_idx] - self._sent_content_idx = sendable_idx - return content - return None - - def _extract_tool_call_jsons(self, text: str) -> list[tuple[str, bool]]: - """Extract (json_text, is_complete) for each region.""" - results: list[tuple[str, bool]] = [] - pos = 0 - while True: - start = text.find(self.tool_call_start_token, pos) - if start == -1: - break - json_start = start + len(self.tool_call_start_token) - json_end = text.find(self.tool_call_end_token, json_start) - if json_end != -1: - results.append((text[json_start:json_end].strip(), True)) - pos = json_end + len(self.tool_call_end_token) - else: - raw = text[json_start:] - # Strip partial suffix if present. - overlap = partial_tag_overlap(raw, self.tool_call_end_token) - if overlap: - raw = raw[:-overlap] - tc_json = raw.strip() - # Valid JSON without closing tag = complete body, - # tag tokens just haven't arrived yet. - is_complete = is_complete_json(tc_json) if tc_json else False - results.append((tc_json, is_complete)) - break - return results - - @staticmethod - def _extract_tool_name(tc_json: str) -> str | None: - """Extract tool name, or None if the name isn't complete yet.""" - match = re.search(r'"name"\s*:\s*"([^"]+)"', tc_json) - return match.group(1) if match else None - - @staticmethod - def _extract_tool_args(tc_json: str, is_complete: bool) -> str | None: - """Extract tool arguments from the tool call JSON. - - Given {"name": "f", "arguments": {"x": 1}}, returns '{"x": 1}'. - When is_complete, strips the trailing '}' that closes the outer - object (not the arguments). For partial JSON, returns as-is. - """ - match = re.search(r'"arguments"\s*:\s*', tc_json) - if not match: - return None - raw = tc_json[match.end() :] - if is_complete: - raw = raw.rstrip() - if raw.endswith("}"): - raw = raw[:-1].rstrip() - return raw - - def _compute_args_diff( - self, index: int, tc_json: str, is_complete: bool - ) -> str | None: - """Return new argument text not yet sent for tool `index`, or None.""" - args = self._extract_tool_args(tc_json, is_complete) - if args is None or len(args) <= len(self.streamed_args_for_tool[index]): - return None - diff = args[len(self.streamed_args_for_tool[index]) :] - self.streamed_args_for_tool[index] = args - self.prev_tool_call_arr[index]["arguments"] = args - return diff - - def extract_tool_calls_streaming( - self, - previous_text: str, - current_text: str, - delta_text: str, - previous_token_ids: Sequence[int], - current_token_ids: Sequence[int], - delta_token_ids: Sequence[int], - request: ChatCompletionRequest, - ) -> DeltaMessage | None: - """Incrementally stream tool call deltas from accumulated output. - - On each invocation, re-parses the full ``current_text`` to find - ```` regions, then diffs against previously sent state - to emit only new content, tool names, or argument fragments. - - Returns a ``DeltaMessage`` containing either plain content (for - text preceding any tool call) or one or more ``DeltaToolCall`` - entries, or ``None`` if there is nothing new to send yet.""" - try: - # Extract any content before tool calls. - content = self._extract_content(current_text) - tool_call_jsons = self._extract_tool_call_jsons(current_text) - tool_call_deltas: list[DeltaToolCall] = [] - - for i, (tc_json, is_complete) in enumerate(tool_call_jsons): - if i >= len(self.prev_tool_call_arr): - self.prev_tool_call_arr.append({}) - self.streamed_args_for_tool.append("") - - # Stream back tool name. - if "name" not in self.prev_tool_call_arr[i]: - name = self._extract_tool_name(tc_json) - if not name: - # Can't skip to tool i+1 if i isn't ready - break - self.prev_tool_call_arr[i]["name"] = name - tool_call_deltas.append( - DeltaToolCall( - index=i, - type="function", - id=make_tool_call_id(), - function=DeltaFunctionCall(name=name).model_dump( - exclude_none=True - ), - ) - ) - - # Stream back new tool args by diffing against what was sent. - args_diff = self._compute_args_diff(i, tc_json, is_complete) - if args_diff: - tool_call_deltas.append( - DeltaToolCall( - index=i, - function=DeltaFunctionCall(arguments=args_diff).model_dump( - exclude_none=True - ), - ) - ) - - if content or tool_call_deltas: - return DeltaMessage( - content=content, - tool_calls=tool_call_deltas, - ) - - return None - - except Exception: - logger.exception("Error trying to handle streaming tool call.") - return None diff --git a/vllm/tool_parsers/hunyuan_a13b_tool_parser.py b/vllm/tool_parsers/hunyuan_a13b_tool_parser.py deleted file mode 100644 index 90e291084f..0000000000 --- a/vllm/tool_parsers/hunyuan_a13b_tool_parser.py +++ /dev/null @@ -1,415 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -# ruff: noqa: E501, SIM102 - -import json -from collections.abc import Sequence -from typing import Any - -import regex as re - -from vllm.entrypoints.generate.base.protocol import ( - DeltaFunctionCall, - DeltaMessage, - DeltaToolCall, - ExtractedToolCallInformation, - FunctionCall, - ToolCall, -) -from vllm.entrypoints.openai.chat_completion.protocol import ( - ChatCompletionRequest, -) -from vllm.logger import init_logger -from vllm.tokenizers import TokenizerLike -from vllm.tool_parsers.abstract_tool_parser import ( - Tool, - ToolParser, -) -from vllm.tool_parsers.utils import consume_space -from vllm.utils import random_uuid - -logger = init_logger(__name__) - - -class HunyuanA13BToolParser(ToolParser): - def __init__(self, tokenizer: TokenizerLike, tools: list[Tool] | None = None): - super().__init__(tokenizer, tools) - - # Initialize state for streaming mode - self.prev_tool_calls: list[dict] = [] - self.current_tool_id = -1 - self.streamed_args: list[str] = [] # Track arguments sent for each tool - - # For backward compatibility with tests - self.current_tools_sent: list[bool] = [] - - # For backward compatibility with serving code - self.prev_tool_call_arr = [] - - # Regex patterns for preprocessing - self.answer_tool_calls_pattern = re.compile( - r"([\s\S]*?)", re.DOTALL - ) - - self.tool_name_reg = re.compile(r'"name"\s*:\s*"([^"]+)"') - - self.tool_empty_arg_reg = re.compile( - r'"name"\s*:\s*"[^"]+"\s*,\s*"arguments"\s*:\s*\{\s*\}' - ) - - # TODO: not support nested json object in fc arguments. - self.tool_non_empty_arg_reg = re.compile( - r'"name"\s*:\s*"[^"]+"\s*,\s*"arguments"\s*:\s*(\{(?:[^{}]|(?:\{[^{}]*\}))*\})' - ) - - self.bot_string = "" - - # Define streaming state type to be initialized later - self.streaming_state: dict[str, Any] = { - "current_tool_index": -1, - "tool_ids": [], - "sent_tools": [], - } - - def preprocess_model_output( - self, model_output: str - ) -> tuple[str | None, str | None]: - # find the location tool call - for match in self.answer_tool_calls_pattern.finditer(model_output): - start, end = match.span() - # check tool_calls whether in side of - think_regions = [ - (m.start(), m.end()) - for m in re.finditer( - r"(.*?)", model_output, flags=re.DOTALL - ) - ] - in_think = any( - start > t_start and end < t_end for t_start, t_end in think_regions - ) - if not in_think: - content = model_output[:start] - tool_calls_content = match.group(1).strip() - try: - json.loads(tool_calls_content) - return content, tool_calls_content - except Exception: - continue - return model_output, None - - def extract_tool_calls( - self, model_output: str, request: ChatCompletionRequest - ) -> ExtractedToolCallInformation: - """Extract tool calls from a complete model output.""" - try: - # Preprocess the model output - content, potential_tool_calls = self.preprocess_model_output(model_output) - - if not potential_tool_calls: - # some text should be filtered out for no function call - # this text is in a13b's chat template. - if content: - content = content.replace("助手:", "", 1) - return ExtractedToolCallInformation( - tools_called=False, tool_calls=[], content=content - ) - - # Parse the potential tool calls as JSON - tool_calls_data = json.loads(potential_tool_calls) - - # Ensure it's an array - if not isinstance(tool_calls_data, list): - logger.debug("Tool calls data is not an array") - return ExtractedToolCallInformation( - tools_called=False, - tool_calls=[], - content=content or model_output, - ) - - tool_calls: list[ToolCall] = [] - - for idx, call in enumerate(tool_calls_data): - if ( - not isinstance(call, dict) - or "name" not in call - or "arguments" not in call - ): - continue - - tool_call = ToolCall( - id=f"call_{random_uuid()}", - type="function", - function=FunctionCall( - name=call["name"], - arguments=( - json.dumps(call["arguments"], ensure_ascii=False) - if isinstance(call["arguments"], dict) - else call["arguments"] - ), - ), - ) - tool_calls.append(tool_call) - - if not content or len(content.strip()) == 0: - # clear the whitespace content. - content = None - - return ExtractedToolCallInformation( - tools_called=len(tool_calls) > 0, - tool_calls=tool_calls, - content=content, - ) - - except Exception: - return ExtractedToolCallInformation( - tools_called=False, tool_calls=[], content=model_output - ) - - def extract_tool_calls_streaming( - self, - previous_text: str, - current_text: str, - delta_text: str, - previous_token_ids: Sequence[int], - current_token_ids: Sequence[int], - delta_token_ids: Sequence[int], - request: ChatCompletionRequest, - ) -> DeltaMessage | None: - """Extract tool calls for streaming mode.""" - start_idx = consume_space(0, current_text) - if current_text[start_idx:].startswith(self.bot_string): - start_idx = consume_space(start_idx + len(self.bot_string), current_text) - if ( - not current_text - or start_idx >= len(current_text) - or current_text[start_idx] != "[" - ): - return DeltaMessage(content=delta_text) - - self._try_parse_json_tools(current_text[start_idx:]) - - test_delta = self._handle_test_compatibility(current_text) - if test_delta: - return test_delta - - name_matches = list(self.tool_name_reg.finditer(current_text)) - tool_count = len(name_matches) - if tool_count == 0: - return None - self._ensure_state_arrays(tool_count) - current_idx = self.streaming_state["current_tool_index"] - - name_delta = self._handle_tool_name_streaming( - current_idx, tool_count, name_matches - ) - if name_delta: - return name_delta - - args_delta = self._handle_tool_args_streaming( - current_text, current_idx, tool_count - ) - if args_delta: - return args_delta - - return None - - def _try_parse_json_tools(self, current_text: str): - try: - parsed_tools = json.loads(current_text) - if isinstance(parsed_tools, list): - self.prev_tool_call_arr = parsed_tools - except json.JSONDecodeError: - pass - - def _handle_test_compatibility(self, current_text: str): - if len(self.current_tools_sent) > 0: - if ( - len(self.current_tools_sent) == 1 - and self.current_tools_sent[0] is False - ): - name_match = self.tool_name_reg.search(current_text) - if name_match: - function_name = name_match.group(1) - tool_id = f"chatcmpl-tool-{random_uuid()}" - delta = DeltaMessage( - tool_calls=[ - DeltaToolCall( - index=0, - type="function", - id=tool_id, - function=DeltaFunctionCall( - name=function_name - ).model_dump(exclude_none=True), - ) - ] - ) - self.current_tools_sent = [True] - self.current_tool_id = 0 - self.streaming_state["current_tool_index"] = 0 - if len(self.streaming_state["sent_tools"]) == 0: - self.streaming_state["sent_tools"].append( - { - "sent_name": True, - "sent_arguments_prefix": False, - "sent_arguments": "", - } - ) - else: - self.streaming_state["sent_tools"][0]["sent_name"] = True - return delta - return None - - def _ensure_state_arrays(self, tool_count: int): - while len(self.streaming_state["sent_tools"]) < tool_count: - self.streaming_state["sent_tools"].append( - { - "sent_name": False, - "sent_arguments_prefix": False, - "sent_arguments": "", - } - ) - while len(self.streaming_state["tool_ids"]) < tool_count: - self.streaming_state["tool_ids"].append(None) - - def _handle_tool_name_streaming( - self, current_idx: int, tool_count: int, name_matches - ): - if current_idx == -1 or current_idx < tool_count - 1: - next_idx = current_idx + 1 - if ( - next_idx < tool_count - and not self.streaming_state["sent_tools"][next_idx]["sent_name"] - ): - self.streaming_state["current_tool_index"] = next_idx - self.current_tool_id = next_idx - current_idx = next_idx - tool_name = name_matches[current_idx].group(1) - tool_id = f"call_{current_idx}_{random_uuid()}" - self.streaming_state["tool_ids"][current_idx] = tool_id - delta = DeltaMessage( - tool_calls=[ - DeltaToolCall( - index=current_idx, - type="function", - id=tool_id, - function=DeltaFunctionCall(name=tool_name).model_dump( - exclude_none=True - ), - ) - ] - ) - self.streaming_state["sent_tools"][current_idx]["sent_name"] = True - while len(self.streamed_args) <= current_idx: - self.streamed_args.append("") - return delta - return None - - def _handle_tool_args_streaming( - self, current_text: str, current_idx: int, tool_count: int - ): - if current_idx >= 0 and current_idx < tool_count: - empty_args_match = self.tool_empty_arg_reg.search(current_text) - if empty_args_match and empty_args_match.start() > 0: - for i in range(tool_count): - if i == current_idx: - if not self.streaming_state["sent_tools"][current_idx][ - "sent_arguments_prefix" - ]: - self.streaming_state["sent_tools"][current_idx][ - "sent_arguments_prefix" - ] = True - self.streaming_state["sent_tools"][current_idx][ - "sent_arguments" - ] = "{}" - while len(self.streamed_args) <= current_idx: - self.streamed_args.append("") - self.streamed_args[current_idx] += "{}" - delta = DeltaMessage( - tool_calls=[ - DeltaToolCall( - index=current_idx, - function=DeltaFunctionCall( - arguments="{}" - ).model_dump(exclude_none=True), - ) - ] - ) - if current_idx < tool_count - 1: - self.streaming_state["current_tool_index"] += 1 - self.current_tool_id = self.streaming_state[ - "current_tool_index" - ] - return delta - - args_matches = list(self.tool_non_empty_arg_reg.finditer(current_text)) - if current_idx < len(args_matches): - args_text = args_matches[current_idx].group(1) - is_last_tool = current_idx == tool_count - 1 - if not is_last_tool: - next_tool_pos = current_text.find( - "},{", args_matches[current_idx].start() - ) - if next_tool_pos != -1: - args_end_pos = next_tool_pos + 1 - args_text = ( - current_text[ - args_matches[current_idx].start() : args_end_pos - ] - .split('"arguments":')[1] - .strip() - ) - sent_args = self.streaming_state["sent_tools"][current_idx][ - "sent_arguments" - ] - if not self.streaming_state["sent_tools"][current_idx][ - "sent_arguments_prefix" - ] and args_text.startswith("{"): - self.streaming_state["sent_tools"][current_idx][ - "sent_arguments_prefix" - ] = True - self.streaming_state["sent_tools"][current_idx][ - "sent_arguments" - ] = "{" - while len(self.streamed_args) <= current_idx: - self.streamed_args.append("") - self.streamed_args[current_idx] += "{" - delta = DeltaMessage( - tool_calls=[ - DeltaToolCall( - index=current_idx, - function=DeltaFunctionCall(arguments="{").model_dump( - exclude_none=True - ), - ) - ] - ) - return delta - - if args_text.startswith(sent_args): - args_diff = args_text[len(sent_args) :] - if args_diff: - self.streaming_state["sent_tools"][current_idx][ - "sent_arguments" - ] = args_text - while len(self.streamed_args) <= current_idx: - self.streamed_args.append("") - self.streamed_args[current_idx] += args_diff - delta = DeltaMessage( - tool_calls=[ - DeltaToolCall( - index=current_idx, - function=DeltaFunctionCall( - arguments=args_diff - ).model_dump(exclude_none=True), - ) - ] - ) - return delta - - if args_text.endswith("}") and args_text == sent_args: - if current_idx < tool_count - 1: - self.streaming_state["current_tool_index"] += 1 - self.current_tool_id = self.streaming_state[ - "current_tool_index" - ] - return None diff --git a/vllm/tool_parsers/hy_v3_tool_parser.py b/vllm/tool_parsers/hy_v3_tool_parser.py deleted file mode 100644 index 9947294ecd..0000000000 --- a/vllm/tool_parsers/hy_v3_tool_parser.py +++ /dev/null @@ -1,697 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -import json -from collections.abc import Sequence -from typing import Any - -import regex as re - -from vllm.entrypoints.generate.base.protocol import ( - DeltaFunctionCall, - DeltaMessage, - DeltaToolCall, - ExtractedToolCallInformation, - FunctionCall, - ToolCall, -) -from vllm.entrypoints.openai.chat_completion.protocol import ( - ChatCompletionRequest, - ChatCompletionToolsParam, -) -from vllm.logger import init_logger -from vllm.renderers.chat_utils import make_tool_call_id -from vllm.tokenizers import TokenizerLike -from vllm.tool_parsers.abstract_tool_parser import ( - Tool, - ToolParser, -) -from vllm.tool_parsers.utils import safe_literal_eval - -logger = init_logger(__name__) - - -class HYV3ToolParser(ToolParser): - _TYPE_ALIASES: dict[str, str] = { - "str": "string", - "text": "string", - "varchar": "string", - "char": "string", - "enum": "string", - "bool": "boolean", - "binary": "boolean", - "int": "integer", - "float": "number", - "double": "number", - "list": "array", - "dict": "object", - "map": "object", - } - - # Prefix-based wildcard matching for non-standard type names. - # Following the same approach as - # qwen3coder_tool_parser._convert_param_value which uses - # param_type.startswith("int"), startswith("uint"), etc. - _INTEGER_PREFIXES: tuple[str, ...] = ( - "int", - "uint", - "long", - "short", - "unsigned", - ) - _NUMBER_PREFIXES: tuple[str, ...] = ("num", "float") - - @staticmethod - def _normalize_type(raw_type: str) -> str: - """Map non-standard type aliases to JSON Schema standard names. - - First performs exact lookup in _TYPE_ALIASES. On miss, falls back - to prefix-based matching using startswith() - - int*/uint*/long*/short*/unsigned* → "integer" - - num*/float* → "number" - """ - exact = HYV3ToolParser._TYPE_ALIASES.get(raw_type) - if exact is not None: - return exact - lower = raw_type.lower() - if any(lower.startswith(p) for p in HYV3ToolParser._INTEGER_PREFIXES): - return "integer" - if any(lower.startswith(p) for p in HYV3ToolParser._NUMBER_PREFIXES): - return "number" - return raw_type - - @staticmethod - def _get_arg_schema( - function_name: str, - arg_key: str, - tools: list[ChatCompletionToolsParam] | None, - ) -> dict: - """Look up a specific argument's property schema from the tools list.""" - if tools is None: - return {} - for tool in tools: - if tool.function.name == function_name: - if tool.function.parameters is None: - return {} - return tool.function.parameters.get("properties", {}).get(arg_key, {}) - logger.warning("No tool named '%s'.", function_name) - return {} - - @staticmethod - def _get_schema_options(arg_schema: dict) -> list[dict]: - """Normalize any property schema into a list of sub-schemas. - - has type (single type) → return [arg_schema] - - anyOf → return the anyOf list - - oneOf → return the oneOf list - - fallback → [{"type": "string"}] - - Note: single ``type`` has the highest priority. - """ - if "type" in arg_schema: - type_val = arg_schema["type"] - # JSON Schema allows "type" to be an array to represent union types, - # e.g. "type": ["string", "object"]. - # Expand it into an anyOf-equivalent format: - # [{"type": "string"}, {"type": "object"}] - # so that _get_types / _parse_value can handle it uniformly later. - if isinstance(type_val, list): - return [{"type": t} for t in type_val] - return [arg_schema] - if "anyOf" in arg_schema: - return arg_schema["anyOf"] - if "oneOf" in arg_schema: - return arg_schema["oneOf"] - - return [{"type": "string"}] - - @staticmethod - def _get_types(arg_schema: dict) -> set[str]: - """Extract normalized, non-null type set from a property schema.""" - schemas = HYV3ToolParser._get_schema_options(arg_schema) - return { - HYV3ToolParser._normalize_type(s.get("type", "string")) for s in schemas - } - {"null"} - - @staticmethod - def _is_only_string_type( - function_name: str, - arg_key: str, - tools: list[ChatCompletionToolsParam] | None, - ) -> bool: - """Return True if the parameter's type set is exactly {"string"}. - - Only pure string types get partial value streaming; compound types - like anyOf(string | array) do not, since the partial value might - end up being a JSON array or object. - """ - arg_schema = HYV3ToolParser._get_arg_schema(function_name, arg_key, tools) - types = HYV3ToolParser._get_types(arg_schema) - return types == {"string"} - - @staticmethod - def _try_parse_bool(value: str) -> bool | None: - """Try to parse a string as bool; return None on failure.""" - lower = value.lower() - if lower == "true": - return True - elif lower == "false": - return False - return None - - @staticmethod - def _try_parse_int(value: str) -> int | None: - """Try to parse a string as int; return None on failure.""" - try: - return int(value) - except (ValueError, TypeError): - return None - - @staticmethod - def _try_parse_wildcard_number(value: str) -> int | float | None: - """Try to parse a string as a number (int or float). - - Decision rule: if the string contains '.' or 'e'/'E' (scientific - notation), parse as float; otherwise parse as int. - - Examples: - "5" → int(5) - "5.0" → float(5.0) - "5.3" → float(5.3) - "1e3" → float(1000.0) - "-3" → int(-3) - - Return None on failure. - - """ - try: - if "." in value or "e" in value or "E" in value: - return float(value) - return int(value) - except (ValueError, TypeError): - return None - - @staticmethod - def _deserialize(value: str) -> Any: - """Deserialize a string value using json.loads then safe_literal_eval.""" - try: - return json.loads(value) - except Exception: - pass - try: - return safe_literal_eval(value) - except Exception: - pass - return value - - @staticmethod - def _parse_value( - value: str, - function_name: str, - arg_key: str, - tools: list[ChatCompletionToolsParam] | None, - ) -> Any: - """Unified argument value parser with anyOf/oneOf support. - - Fallthrough chain: - bool → int → number(wildcard_number) - → json.loads for array/object - → string → _deserialize - """ - arg_schema = HYV3ToolParser._get_arg_schema(function_name, arg_key, tools) - types = HYV3ToolParser._get_types(arg_schema) - - # 1. Try bool - if "boolean" in types: - result_bool = HYV3ToolParser._try_parse_bool(value) - if result_bool is not None: - return result_bool - - # 2. Try int - if "integer" in types: - result_int = HYV3ToolParser._try_parse_int(value) - if result_int is not None: - return result_int - - # 3. Try number (wildcard_number: int if no '.'/e/E, float otherwise) - if "number" in types: - result_number = HYV3ToolParser._try_parse_wildcard_number(value) - if result_number is not None: - return result_number - - # 4. Try json.loads (covers array/object and other unlisted types) - if types - {"string", "boolean", "integer", "number"}: - try: - return json.loads(value) - except (json.JSONDecodeError, ValueError): - pass - - # 5. String fallback - if "string" in types: - return value - - # 6. Final fallback - return HYV3ToolParser._deserialize(value) - - def __init__(self, tokenizer: TokenizerLike, tools: list[Tool] | None = None): - super().__init__(tokenizer, tools) - - self.prev_tool_call_arr: list[dict] = [] - self.current_tool_id: int = -1 - self.streamed_args_for_tool: list[ - str - ] = [] # map what has been streamed for each tool so far to a list - - # Streaming state: send tool name first, then return arguments at once - self._streaming_tool_name: str | None = None # tool name being streamed - - # State fields for incremental argument streaming - self._completed_args: dict = {} # closed {key: parsed_value} - self._current_arg_key: str | None = None # key being collected - self._current_arg_is_string: bool = False # is current arg pure string? - self._streamed_json_len: int = 0 # bytes of JSON already sent - - init_kwargs = getattr(tokenizer, "init_kwargs", None) or {} - self.suffix: str = init_kwargs.get("token_suffix") or "" - - self.tool_calls_start_token: str = f"" - self.tool_calls_end_token: str = f"" - - self.tool_call_start_token: str = f"" - self.tool_call_end_token: str = f"" - - self.tool_sep_token: str = f"" - - self.arg_key_start_token: str = f"" - self.arg_key_end_token: str = f"" - - self.arg_value_start_token: str = f"" - self.arg_value_end_token: str = f"" - - self.tool_call_regex = re.compile( - rf"{self.tool_call_start_token}(.*?){self.tool_sep_token}" - rf"(.*?){self.tool_call_end_token}", - re.DOTALL, - ) - - self.tool_call_portion_regex = re.compile( - rf"{self.tool_call_start_token}(.*?){self.tool_sep_token}(.*)", re.DOTALL - ) - - self.func_args_regex = re.compile( - rf"{self.arg_key_start_token}(.*?){self.arg_key_end_token}\s*" - rf"{self.arg_value_start_token}(.*?){self.arg_value_end_token}", - re.DOTALL, - ) - - if not self.model_tokenizer: - raise ValueError( - "The model tokenizer must be passed to the ToolParser " - "constructor during construction." - ) - self.tool_calls_start_token_id = self.vocab.get(self.tool_calls_start_token) - self.tool_calls_end_token_id = self.vocab.get(self.tool_calls_end_token) - - self.tool_call_start_token_id = self.vocab.get(self.tool_call_start_token) - self.tool_call_end_token_id = self.vocab.get(self.tool_call_end_token) - self._buffer = "" - - if ( - self.tool_calls_start_token_id is None - or self.tool_calls_end_token_id is None - ): - raise RuntimeError( - "HYV3 Tool parser could not locate tool call " - "start/end tokens in the tokenizer!" - ) - - def _extract_tool_calls( - self, - model_output: str, - request: ChatCompletionRequest, - ) -> list[ToolCall]: - try: - function_call_tuples = [] - # start_token{name}sep_token{args}end_token... - function_calls = self.tool_call_regex.findall(model_output) - if function_calls: - function_call_tuples.extend(function_calls) - remaining = model_output.split(self.tool_call_end_token)[-1] - function_calls = self.tool_call_portion_regex.findall(remaining) - function_call_tuples += function_calls - else: - function_calls = self.tool_call_portion_regex.findall(model_output) - if function_calls: - function_call_tuples.extend(function_calls) - tool_calls = [] - for match in function_call_tuples: - function_name, function_args = match - function_name = function_name.strip() - function_args = function_args.strip() - - arg_pairs = self.func_args_regex.findall(function_args) - arg_dict = {} - for key, value in arg_pairs: - parsed_value = HYV3ToolParser._parse_value( - value, function_name, key, request.tools - ) - arg_dict[key] = parsed_value - tool_calls.append( - ToolCall( - type="function", - function=FunctionCall( - name=function_name, - arguments=json.dumps(arg_dict, ensure_ascii=False), - ), - ) - ) - return tool_calls - except Exception: - logger.exception("Error in extracting tool call from response.") - return [] - - def extract_tool_calls( - self, - model_output: str, - request: ChatCompletionRequest, - ) -> ExtractedToolCallInformation: - # sanity check; avoid unnecessary processing - if self.tool_calls_start_token not in model_output: - return ExtractedToolCallInformation( - tools_called=False, tool_calls=[], content=model_output - ) - else: - try: - tool_calls = self._extract_tool_calls(model_output, request) - - s_index = model_output.find(self.tool_calls_start_token) - content = model_output[:s_index] if s_index != -1 else model_output - return ExtractedToolCallInformation( - tools_called=True, - tool_calls=tool_calls, - content=content if content else None, - ) - - except Exception: - logger.exception("Error in extracting tool call from response.") - return ExtractedToolCallInformation( - tools_called=False, tool_calls=[], content=model_output - ) - - def _reset_streaming_tool_state(self): - """Reset the streaming state for a single tool call.""" - self._streaming_tool_name = None - self._completed_args = {} - self._current_arg_key = None - self._current_arg_is_string = False - self._streamed_json_len = 0 - - def extract_tool_calls_streaming( - self, - previous_text: str, - current_text: str, - delta_text: str, - previous_token_ids: Sequence[int], - current_token_ids: Sequence[int], - delta_token_ids: Sequence[int], - request: ChatCompletionRequest, - ) -> DeltaMessage | None: - # Check whether current tokens contain the tool_calls start token - if self.tool_calls_start_token_id not in current_token_ids: - return DeltaMessage(content=delta_text) - - # Encountered tool_calls start tag; extract preceding content and buffer - content_delta: str | None = None - if self.tool_calls_start_token in delta_text: - text_parts = delta_text.split(self.tool_calls_start_token) - self._buffer += text_parts[-1] - content_delta = text_parts[0] or None - else: - self._buffer += delta_text - - # Encountered finish, extract valid arguments - if ( - current_text.find(self.tool_call_end_token + self.tool_calls_end_token) - != -1 - and self._buffer.find(self.tool_call_end_token) == -1 - ): - self._buffer += self.tool_call_end_token + self.tool_calls_end_token - - cur_text = self._buffer - - # Haven't encountered tool_call start tag yet; keep buffering - start_idx = cur_text.find(self.tool_call_start_token) - if start_idx == -1 and self._streaming_tool_name is None: - if content_delta is not None: - return DeltaMessage(content=content_delta) - self._buffer = "" - return None - - # === Phase 1: Detect tool name (send when tool_sep_token is seen) === - name_delta: DeltaMessage | None = None - if self._streaming_tool_name is None: - sep_idx = cur_text.find(self.tool_sep_token) - if sep_idx == -1: - # tool_sep not yet seen; keep buffering from tool_call_start - self._buffer = cur_text[start_idx:] - if content_delta is None: - return None - return DeltaMessage(content=content_delta) - - # Extract tool name: between tool_call_start_token and tool_sep_token - name_start = start_idx + len(self.tool_call_start_token) - tool_name = cur_text[name_start:sep_idx].strip() - self._streaming_tool_name = tool_name - - # Update buffer: keep only content after tool_sep (i.e. the args portion) - self._buffer = cur_text[sep_idx + len(self.tool_sep_token) :] - - # Increment tool_id and send a chunk containing only the name - self.current_tool_id += 1 - self._current_tool_call_id = make_tool_call_id() - name_delta = DeltaMessage( - tool_calls=[ - DeltaToolCall( - index=self.current_tool_id, - id=self._current_tool_call_id, - type="function", - function=DeltaFunctionCall( - name=tool_name, - ), - ) - ] - ) - name_delta.content = content_delta - - # Check if buffer already has complete arguments (all-in-one-delta) - if self.tool_call_end_token not in self._buffer: - return name_delta - # Buffer already has a complete tool call; continue to phase 2 below - - # === Phase 2: Incremental argument streaming === - return self._extract_streaming_incremental(name_delta, request) - - def _make_args_delta(self, argument_diff: str) -> DeltaMessage: - """Build a DeltaMessage containing only an arguments diff.""" - return DeltaMessage( - tool_calls=[ - DeltaToolCall( - index=self.current_tool_id, - function=DeltaFunctionCall(arguments=argument_diff), - ) - ] - ) - - def _drain_calls(self, request: ChatCompletionRequest) -> list[DeltaToolCall]: - deltas: list[DeltaToolCall] = [] - while (start_idx := self._buffer.find(self.tool_call_start_token)) != -1: - end_idx = self._buffer.find(self.tool_call_end_token, start_idx) - if end_idx == -1: - break - end_idx += len(self.tool_call_end_token) - complete_call = self._buffer[start_idx:end_idx] - self._buffer = self._buffer[end_idx:] - for call in self._extract_tool_calls(complete_call, request): - self.current_tool_id += 1 - arguments = call.function.arguments - self.prev_tool_call_arr.append( - {"name": call.function.name, "arguments": json.loads(arguments)} - ) - self.streamed_args_for_tool.append(arguments) - deltas.append( - DeltaToolCall( - index=self.current_tool_id, - id=make_tool_call_id(), - type="function", - function=DeltaFunctionCall( - name=call.function.name, arguments=arguments - ), - ) - ) - return deltas - - def _extract_streaming_incremental( - self, - name_delta: DeltaMessage | None, - request: ChatCompletionRequest, - ) -> DeltaMessage | None: - """Incremental phase-2: scan tags in buffer, emit JSON diffs. - - Strategy: - - Track completed args and emit each one as a JSON fragment. - - For string-typed args, stream the value character-by-character. - - Withhold the closing ``}`` until ```` is seen. - - We build JSON manually via fragments rather than using json.dumps - with a cursor, because json.dumps of partial-vs-full string values - produces incompatible prefixes (e.g. ``""}`` vs ``"Hello"}``). - """ - buf = self._buffer - is_complete = self.tool_call_end_token in buf - - if is_complete: - end_idx = buf.find(self.tool_call_end_token) - args_text = buf[:end_idx] - remaining = buf[end_idx + len(self.tool_call_end_token) :] - else: - args_text = buf - remaining = "" - - # --- scan all fully closed kv pairs --- - arg_pairs = self.func_args_regex.findall(args_text) - for key, value in arg_pairs: - key = key.strip() - if key not in self._completed_args: - parsed_value = HYV3ToolParser._parse_value( - value, self._streaming_tool_name or "", key, request.tools - ) - self._completed_args[key] = parsed_value - - # --- detect partial (unclosed) kv at the tail --- - last_closed_end = 0 - for m in self.func_args_regex.finditer(args_text): - last_closed_end = m.end() - tail = args_text[last_closed_end:] - - partial_key: str | None = None - partial_value: str | None = None - - ak_start = tail.find(self.arg_key_start_token) - if ak_start != -1: - ak_end = tail.find( - self.arg_key_end_token, - ak_start + len(self.arg_key_start_token), - ) - if ak_end != -1: - partial_key = tail[ - ak_start + len(self.arg_key_start_token) : ak_end - ].strip() - self._current_arg_key = partial_key - self._current_arg_is_string = HYV3ToolParser._is_only_string_type( - self._streaming_tool_name or "", - partial_key, - request.tools, - ) - - av_start = tail.find(self.arg_value_start_token, ak_end) - if av_start != -1: - val_content_start = av_start + len(self.arg_value_start_token) - if self._current_arg_is_string: - partial_value = tail[val_content_start:] - else: - # key not yet closed - self._current_arg_key = None - self._current_arg_is_string = False - - # --- build the current JSON snapshot as a string --- - # We construct JSON manually so we can precisely control - # what gets sent incrementally. - snapshot_parts: list[str] = [] - for k, v in self._completed_args.items(): - k_json = json.dumps(k, ensure_ascii=False) - v_json = json.dumps(v, ensure_ascii=False) - snapshot_parts.append(f"{k_json}: {v_json}") - - if partial_key is not None and partial_value is not None: - k_json = json.dumps(partial_key, ensure_ascii=False) - # For string partial value, we build the JSON string - # WITHOUT the closing quote, so the prefix stays stable - # as the value grows. The closing `"` and `}` will be - # sent when the value or tool_call closes. - escaped_val = ( - partial_value.replace("\\", "\\\\") - .replace('"', '\\"') - .replace("\n", "\\n") - .replace("\r", "\\r") - .replace("\t", "\\t") - ) - # Note: no closing " here – it's appended only on close - snapshot_parts.append(f'{k_json}: "{escaped_val}') - - snapshot = "{" + ", ".join(snapshot_parts) + "}" - - # --- compute diff --- - argument_diff: str | None = None - - if is_complete: - # Tool call finished – send everything remaining. - # Build final snapshot with proper JSON (all values closed). - final_args = dict(self._completed_args) - final_json = json.dumps(final_args, ensure_ascii=False) - if self._streamed_json_len < len(final_json): - argument_diff = final_json[self._streamed_json_len :] - self._streamed_json_len = len(final_json) - - # Record into prev_tool_call_arr - self.prev_tool_call_arr.append( - { - "name": self._streaming_tool_name, - "arguments": final_args, - } - ) - self.streamed_args_for_tool.append(final_json) - - self._reset_streaming_tool_state() - self._buffer = remaining - else: - # Still in progress – withhold the tail. - # For open strings: snapshot ends with ...partial_val} - # we withhold "}" (1 char) – the missing closing " will - # be sent when the value closes. - # For no open string: snapshot ends with ...value"} - # we withhold "}" (1 char). - end = len(snapshot) - 1 # exclude trailing "}" - if end > self._streamed_json_len: - argument_diff = snapshot[self._streamed_json_len : end] - self._streamed_json_len = end - - # --- construct return DeltaMessage --- - delta: DeltaMessage | None - if name_delta is not None and argument_diff: - nd_func = name_delta.tool_calls[0].function - delta = DeltaMessage( - tool_calls=[ - DeltaToolCall( - index=self.current_tool_id, - id=self._current_tool_call_id, - type="function", - function=DeltaFunctionCall( - name=nd_func.name if nd_func else None, - arguments=argument_diff, - ), - ) - ] - ) - delta.content = name_delta.content - elif name_delta is not None: - delta = name_delta - elif argument_diff: - delta = self._make_args_delta(argument_diff) - else: - delta = None - - extra_tool_calls = self._drain_calls(request) if is_complete else [] - if extra_tool_calls: - if delta is None: - return DeltaMessage(tool_calls=extra_tool_calls) - delta.tool_calls.extend(extra_tool_calls) - return delta diff --git a/vllm/tool_parsers/hy_v4_tool_parser.py b/vllm/tool_parsers/hy_v4_tool_parser.py deleted file mode 100644 index de707c0022..0000000000 --- a/vllm/tool_parsers/hy_v4_tool_parser.py +++ /dev/null @@ -1,1174 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from __future__ import annotations - -import ast -import json -from collections.abc import Sequence -from typing import TYPE_CHECKING, Any, TypedDict - -import regex as re - -import vllm.envs as envs -from vllm.entrypoints.generate.base.protocol import ( - DeltaFunctionCall, - DeltaMessage, - DeltaToolCall, - ExtractedToolCallInformation, - FunctionCall, - ToolCall, -) -from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest -from vllm.entrypoints.openai.responses.protocol import ResponsesRequest -from vllm.logger import init_logger -from vllm.renderers.chat_utils import make_tool_call_id -from vllm.tokenizers import TokenizerLike -from vllm.tool_parsers.abstract_tool_parser import ToolParser -from vllm.tool_parsers.tool_strict_level import ToolStrictLevel -from vllm.tool_parsers.utils import partial_tag_overlap - -if TYPE_CHECKING: - from xgrammar import StructuralTag - -logger = init_logger(__name__) - -# ============================================================================= -# Plain-dict contract with the vLLM-side wrapper -# ============================================================================= - - -class ToolSchema(TypedDict): - """A tool as seen by this parser (built by the wrapper from ``request.tools``).""" - - name: str - parameters: dict[str, Any] | None # JSON Schema, or None - - -class ToolCallDict(TypedDict): - """One extracted tool call. ``arguments`` is a JSON-encoded string.""" - - name: str - arguments: str - - -class ExtractResult(TypedDict): - """Return of ``extract_tool_calls`` (non-streaming).""" - - tools_called: bool - content: str | None - tool_calls: list[ToolCallDict] - - -class StreamToolCall(TypedDict): - """One streaming tool-call delta. - - ``new`` is True only on the chunk that first emits this tool's name -- the - wrapper mints an id + sets type on that chunk only. ``arguments`` is the - incremental JSON diff (or None). - """ - - index: int - new: bool - name: str | None - arguments: str | None - - -class StreamDelta(TypedDict): - """Return of ``extract_tool_calls_streaming``. - - A single engine delta can carry content plus every tool call that was - drained from the buffer (batched decode), so ``tool_calls`` is not limited - to one entry. - """ - - content: str | None - tool_calls: list[StreamToolCall] - - -# NOTE: mirrored in ``vllm.reasoning.hy_v4_reasoning_parser`` (same pattern as -# ``gemma4_utils``) so neither package depends on the other. -def detect_token_suffix(tokenizer: TokenizerLike) -> str: - """Detect the per-checkpoint suffix used by Hunyuan structural tokens. - - Args: - tokenizer: Tokenizer of the served checkpoint. - - Returns: - The suffix including its leading colon (e.g. ``":6124c78e"``), or ``""`` - when the checkpoint uses unsuffixed tokens. - - Raises: - RuntimeError: The tokenizer declares the structural tokens through - ``model_specific_special_tokens``, which transformers 5 no longer - round-trips. - - """ - init_kwargs = getattr(tokenizer, "init_kwargs", None) or {} - think_begin_as_special = init_kwargs.get("model_specific_special_tokens", {}).get( - "think_begin_token", "" - ) - if think_begin_as_special: - raise RuntimeError( - "This checkpoint declares HYV4 structural tokens (think_begin_token" - "/toolcalls_begin_token/argkey_begin_token) in " - "tokenizer_config.json, which transformers 5 no longer supports. " - "Remove those fields and keep the tokens in the tokenizer's own " - "token definitions so the suffix can be read from the vocab." - ) - - structural_token_re = re.compile( - r"<(?:think|tool_calls|tool_call|arg_key|arg_value)(:[^\s>]+)?>" - ) - for token in tokenizer.get_vocab(): - match = structural_token_re.fullmatch(token) - if match: - return match.group(1) or "" - - return "" - - -# ============================================================================= -# Argument-value parsing utilities (stateless, pure functions) -# -# These map the model's raw ```` text to typed Python values using -# the tool's JSON Schema. They do not touch any parser state, so they live at -# module level instead of tangling up HYV4ToolExtractor. -# ============================================================================= - -_TYPE_ALIASES: dict[str, str] = { - "str": "string", - "text": "string", - "varchar": "string", - "char": "string", - "enum": "string", - "bool": "boolean", - "binary": "boolean", - "int": "integer", - "float": "number", - "double": "number", - "list": "array", - "dict": "object", - "map": "object", -} - -# Prefix-based wildcard matching for non-standard type names. -# Following the same approach as qwen3coder_tool_parser._convert_param_value -# which uses param_type.startswith("int"), startswith("uint"), etc. -_INTEGER_PREFIXES: tuple[str, ...] = ("int", "uint", "long", "short", "unsigned") -_NUMBER_PREFIXES: tuple[str, ...] = ("num", "float") - - -def _normalize_type(raw_type: str) -> str: - """Map non-standard type aliases to JSON Schema standard names. - - First performs exact lookup in _TYPE_ALIASES. On miss, falls back to - prefix-based matching using startswith() - - int*/uint*/long*/short*/unsigned* → "integer" - - num*/float* → "number" - """ - exact = _TYPE_ALIASES.get(raw_type) - if exact is not None: - return exact - lower = raw_type.lower() - if any(lower.startswith(p) for p in _INTEGER_PREFIXES): - return "integer" - if any(lower.startswith(p) for p in _NUMBER_PREFIXES): - return "number" - return raw_type - - -def _get_arg_schema( - function_name: str, - arg_key: str, - tools: list[ToolSchema] | None, -) -> dict[str, Any]: - """Look up a specific argument's property schema from the tools list.""" - if tools is None: - return {} - for tool in tools: - if tool.get("name") == function_name: - parameters = tool.get("parameters") - if parameters is None: - return {} - return parameters.get("properties", {}).get(arg_key, {}) - logger.warning_once("No tool named '%s'.", function_name) - return {} - - -def _get_schema_options(arg_schema: dict[str, Any]) -> list[dict[str, Any]]: - """Normalize any property schema into a list of sub-schemas. - - has type (single type) → return [arg_schema] - - anyOf → return the anyOf list - - oneOf → return the oneOf list - - fallback → [{"type": "string"}] - - Note: single ``type`` has the highest priority. - """ - if "type" in arg_schema: - type_val = arg_schema["type"] - # JSON Schema allows "type" to be an array to represent union types, - # e.g. "type": ["string", "object"]. - # Expand it into an anyOf-equivalent format: - # [{"type": "string"}, {"type": "object"}] - # so that _get_types / _parse_value can handle it uniformly later. - if isinstance(type_val, list): - return [{"type": t} for t in type_val] - return [arg_schema] - if "anyOf" in arg_schema: - return arg_schema["anyOf"] - if "oneOf" in arg_schema: - return arg_schema["oneOf"] - - return [{"type": "string"}] - - -def _get_types(arg_schema: dict[str, Any]) -> set[str]: - """Extract normalized, non-null type set from a property schema.""" - schemas = _get_schema_options(arg_schema) - return {_normalize_type(s.get("type", "string")) for s in schemas} - {"null"} - - -def _is_only_string_type( - function_name: str, - arg_key: str, - tools: list[ToolSchema] | None, -) -> bool: - """Return True if the parameter's type set is exactly {"string"}. - - Only pure string types get partial value streaming; compound types like - anyOf(string | array) do not, since the partial value might end up being a - JSON array or object. - """ - types = _get_types(_get_arg_schema(function_name, arg_key, tools)) - return types == {"string"} - - -def _try_parse_bool(value: str) -> bool | None: - """Try to parse a string as bool; return None on failure.""" - lower = value.lower() - if lower == "true": - return True - elif lower == "false": - return False - return None - - -def _try_parse_int(value: str) -> int | None: - """Try to parse a string as int; return None on failure.""" - try: - return int(value) - except (ValueError, TypeError): - return None - - -def _try_parse_wildcard_number(value: str) -> int | float | None: - """Try to parse a string as a number (int or float). - - Decision rule: if the string contains '.' or 'e'/'E' (scientific notation), - parse as float; otherwise parse as int. - - Examples: "5" → 5, "5.0" → 5.0, "1e3" → 1000.0, "-3" → -3. - Return None on failure. - """ - try: - if "." in value or "e" in value or "E" in value: - return float(value) - return int(value) - except (ValueError, TypeError): - return None - - -def _deserialize(value: str) -> Any: - """Deserialize a string value using json.loads then ast.literal_eval.""" - try: - return json.loads(value) - except Exception: - pass - try: - return ast.literal_eval(value) - except Exception: - pass - return value - - -def _parse_value( - value: str, - function_name: str, - arg_key: str, - tools: list[ToolSchema] | None, -) -> Any: - """Unified argument value parser with anyOf/oneOf support. - - Fallthrough chain: - bool → int → number(wildcard_number) - → json.loads for array/object → string → _deserialize - """ - types = _get_types(_get_arg_schema(function_name, arg_key, tools)) - - # 1. Try bool - if "boolean" in types: - result_bool = _try_parse_bool(value) - if result_bool is not None: - return result_bool - - # 2. Try int - if "integer" in types: - result_int = _try_parse_int(value) - if result_int is not None: - return result_int - - # 3. Try number (wildcard_number: int if no '.'/e/E, float otherwise) - if "number" in types: - result_number = _try_parse_wildcard_number(value) - if result_number is not None: - return result_number - - # 4. Try json.loads (covers array/object and other unlisted types) - if types - {"string", "boolean", "integer", "number"}: - try: - return json.loads(value) - except (json.JSONDecodeError, ValueError): - pass - - # 5. String fallback - if "string" in types: - return value - - # 6. Final fallback - return _deserialize(value) - - -# ============================================================================= -# Stateful tool-call parser -# ============================================================================= - - -class HYV4ToolExtractor: - """Pure tool-call parsing logic for HYV4. Returns plain dicts; no vLLM types. - - Holds only parser state: the structural token strings/ids, the compiled - regexes, and the streaming incremental state. Stateless argument parsing is - delegated to the module-level utilities above. - - Streaming has two entry paths sharing one incremental argument parser, - selected by the ``guided`` flag of :meth:`extract_tool_calls_streaming`: - normal / auto tool choice detects markers on atomic token ids so ordinary - ``<`` in content is never withheld, while structural-tag guided decoding - detects them on the string level because the grammar may split a marker - across sub-word tokens. - """ - - def __init__(self, vocab: dict[str, int], token_suffix: str, strict: bool): - self._strict = strict - - # Kept so the wrapper can build a matching structural tag for - # constrained decoding without re-scanning the tokenizer vocab. - self.token_suffix: str = token_suffix - - # Running state read by the vLLM serving layer (mirrored by the wrapper). - self.prev_tool_call_arr: list[dict] = [] - self.current_tool_id: int = -1 - self.streamed_args_for_tool: list[str] = [] - - # Streaming state: send tool name first, then return arguments at once - self._streaming_tool_name: str | None = None # tool name being streamed - - # State fields for incremental argument streaming - self._completed_args: dict = {} # closed {key: parsed_value} - self._current_arg_key: str | None = None # key being collected - self._current_arg_is_string: bool = False # is current arg pure string? - self._streamed_json_len: int = 0 # bytes of JSON already sent - - self.tool_calls_start_token: str = f"" - self.tool_calls_end_token: str = f"" - - self.tool_call_start_token: str = f"" - self.tool_call_end_token: str = f"" - - self.arg_key_start_token: str = f"" - self.arg_key_end_token: str = f"" - - self.arg_value_start_token: str = f"" - self.arg_value_end_token: str = f"" - - logger.debug( - "HYV4ToolExtractor structural tokens: %s ... %s, suffix=%r, strict=%s", - self.tool_calls_start_token, - self.tool_calls_end_token, - token_suffix, - strict, - ) - - # The suffix comes from the tokenizer vocab, so it must not be treated - # as a pattern. - self.tool_call_regex = re.compile( - rf"{re.escape(self.tool_call_start_token)}(.*?)" - rf"{re.escape(self.tool_call_end_token)}", - re.DOTALL, - ) - - self.func_args_regex = re.compile( - rf"{re.escape(self.arg_key_start_token)}(.*?)" - rf"{re.escape(self.arg_key_end_token)}" - rf"{re.escape(self.arg_value_start_token)}(.*?)" - rf"{re.escape(self.arg_value_end_token)}", - re.DOTALL, - ) - - self.tool_calls_start_token_id = vocab.get(self.tool_calls_start_token) - self.tool_calls_end_token_id = vocab.get(self.tool_calls_end_token) - - self.tool_call_start_token_id = vocab.get(self.tool_call_start_token) - self.tool_call_end_token_id = vocab.get(self.tool_call_end_token) - self._buffer = "" - - if ( - self.tool_calls_start_token_id is None - or self.tool_calls_end_token_id is None - ): - raise RuntimeError( - "HYV4 tool extractor could not locate tool call " - "start/end tokens in the tokenizer!" - ) - - def _extract_tool_calls( - self, - model_output: str, - tools: list[ToolSchema] | None, - ) -> list[ToolCallDict]: - try: - tool_calls: list[ToolCallDict] = [] - function_calls = self.tool_call_regex.findall(model_output) - for tool_call_body in function_calls: - arg_start = tool_call_body.find(self.arg_key_start_token) - if arg_start == -1: - function_name = tool_call_body.strip() - function_args = "" - else: - function_name = tool_call_body[:arg_start].strip() - function_args = tool_call_body[arg_start:].strip() - - arg_pairs = self.func_args_regex.findall(function_args) - arg_dict = {} - for key, value in arg_pairs: - arg_dict[key] = _parse_value(value, function_name, key, tools) - tool_calls.append( - ToolCallDict( - name=function_name, - arguments=json.dumps(arg_dict, ensure_ascii=False), - ) - ) - return tool_calls - except Exception: - logger.exception("Error in extracting tool call from response.") - return [] - - def _extract_tool_calls_strict( - self, - model_output: str, - tools: list[ToolSchema] | None, - ) -> list[ToolCallDict]: - start_idx = model_output.find(self.tool_calls_start_token) - end_idx = model_output.find(self.tool_calls_end_token) - if start_idx == -1 or end_idx == -1 or end_idx < start_idx: - raise ValueError("Malformed tool block wrapper.") - - tool_block = model_output[ - start_idx + len(self.tool_calls_start_token) : end_idx - ] - if tool_block.count(self.tool_call_start_token) != tool_block.count( - self.tool_call_end_token - ): - raise ValueError("Malformed tool_call wrapper count.") - - tool_calls: list[ToolCallDict] = [] - function_calls = self.tool_call_regex.findall(tool_block) - if tool_block.count(self.tool_call_start_token) != len(function_calls): - raise ValueError("Malformed tool_call body.") - - for tool_call_body in function_calls: - arg_start = tool_call_body.find(self.arg_key_start_token) - if arg_start == -1: - function_name = tool_call_body.strip() - function_args = "" - else: - function_name = tool_call_body[:arg_start].strip() - function_args = tool_call_body[arg_start:].strip() - - if not function_name: - raise ValueError("Empty function name in tool call.") - - if function_args.count(self.arg_key_start_token) != function_args.count( - self.arg_key_end_token - ) or function_args.count(self.arg_value_start_token) != function_args.count( - self.arg_value_end_token - ): - raise ValueError("Malformed argument tag count.") - - arg_pairs = self.func_args_regex.findall(function_args) - remainder = self.func_args_regex.sub("", function_args) - if remainder.strip(): - raise ValueError("Unparsed argument payload remains.") - - arg_dict = {} - for key, value in arg_pairs: - arg_dict[key] = _parse_value(value, function_name, key, tools) - - tool_calls.append( - ToolCallDict( - name=function_name, - arguments=json.dumps(arg_dict, ensure_ascii=False), - ) - ) - - if not tool_calls: - raise ValueError("No valid tool calls extracted.") - - return tool_calls - - def extract_tool_calls( - self, - model_output: str, - tools: list[ToolSchema] | None, - ) -> ExtractResult: - # sanity check; avoid unnecessary processing - if self.tool_calls_start_token not in model_output: - return ExtractResult( - tools_called=False, content=model_output, tool_calls=[] - ) - else: - try: - if self._strict: - tool_calls = self._extract_tool_calls_strict(model_output, tools) - else: - tool_calls = self._extract_tool_calls(model_output, tools) - if not tool_calls: - raise ValueError("No valid tool calls extracted.") - - s_index = model_output.find(self.tool_calls_start_token) - content = model_output[:s_index] if s_index != -1 else model_output - return ExtractResult( - tools_called=True, - content=content if content else None, - tool_calls=tool_calls, - ) - - except Exception: - logger.warning( - "HYV4ToolExtractor detected malformed tool output; " - "falling back to raw content." - ) - return ExtractResult( - tools_called=False, - content=model_output, - tool_calls=[], - ) - - def _reset_streaming_tool_state(self): - """Reset the streaming state for a single tool call.""" - self._streaming_tool_name = None - self._completed_args = {} - self._current_arg_key = None - self._current_arg_is_string = False - self._streamed_json_len = 0 - - def extract_tool_calls_streaming( - self, - previous_text: str, - current_text: str, - delta_text: str, - previous_token_ids: Sequence[int], - current_token_ids: Sequence[int], - delta_token_ids: Sequence[int], - tools: list[ToolSchema] | None, - *, - guided: bool = False, - ) -> StreamDelta | None: - """Consume one engine delta and emit content / tool-call deltas. - - Two entry paths share the same incremental argument parser: - - ``guided=False`` (normal / auto tool choice): the structural markers - arrive as atomic tokenizer tokens, so detection is done on token ids - and ordinary ``<`` in model content is never withheld. - - ``guided=True`` (structural-tag constrained decoding): the grammar - only constrains strings, so ```` may be split across - sub-word tokens and must be detected on the string level. - - Args: - previous_text: Text decoded before this delta. - current_text: Full text decoded so far. - delta_text: Text decoded in this delta. - previous_token_ids: Token ids before this delta. - current_token_ids: Token ids decoded so far. - delta_token_ids: Token ids in this delta. - tools: Plain tool schemas used for argument typing. - guided: Whether guided decoding may split markers into sub-word - tokens. - - Returns: - A streaming delta carrying content and/or the tool calls drained - from the buffer, or None when nothing can be emitted yet. - - """ - content_delta: str | None = None - tool_calls: list[StreamToolCall] = [] - - if guided: - # Guided path: a structural-tag grammar constrains strings, not - # token ids, so the atomic ``tool_calls_start_token_id`` never - # appears in the token stream and an id-based check would - # misclassify every tool delta as plain content. - marker = self.tool_calls_start_token - marker_pos = current_text.find(marker) - - if marker_pos == -1: - # Marker not seen yet. The tail of ``current_text`` might still - # be a partial prefix of the marker split across deltas; hold - # that tail back and only stream the safe portion as content. - safe_end = len(current_text) - partial_tag_overlap(current_text, marker) - # Emit only the newly-arrived, safe content (delta relative to - # what was previously streamed). - content = current_text[len(previous_text) : safe_end] - return StreamDelta( - content=content if content else None, - tool_calls=[], - ) - - # Marker present. Stream any content that precedes it (only the - # part we have not already emitted), then buffer the tool payload. - # - # The payload is everything after the marker in the *full* text. We - # keep the buffer in sync with it by only appending the portion that - # arrived in this delta, mirroring the incremental ``+=`` semantics - # while remaining correct when the marker itself was split across - # sub-word tokens. - content_end = marker_pos - content = current_text[len(previous_text) : content_end] - - payload_start = marker_pos + len(marker) - # How much of the payload had already been buffered before this - # delta. - prev_payload_len = max(len(previous_text) - payload_start, 0) - new_payload = current_text[payload_start + prev_payload_len :] - self._buffer += new_payload - - if content: - content_delta = content - else: - # Atomic-token path: normal / auto generation emits the HYV4 - # structural markers as single tokenizer tokens. Pass ordinary - # content through exactly as decoded, including comparison - # operators such as ``<``. - if self.tool_calls_start_token_id not in current_token_ids: - return StreamDelta(content=delta_text, tool_calls=[]) - - if self.tool_calls_start_token in delta_text: - text_parts = delta_text.split(self.tool_calls_start_token) - self._buffer += text_parts[-1] - if text_parts[0]: - content_delta = text_parts[0] - else: - self._buffer += delta_text - - # Encountered finish, extract valid arguments - if ( - current_text.find(self.tool_call_end_token + self.tool_calls_end_token) - != -1 - and self._buffer.find(self.tool_call_end_token) == -1 - ): - self._buffer += self.tool_call_end_token + self.tool_calls_end_token - - # Drain every complete tool call that is already buffered so a single - # engine delta can carry content + N tool calls (batched decode). - while True: - cur_text = self._buffer - - # Haven't encountered tool_call start tag yet; drop the inter-tag - # padding but keep any tail that may be a partial tool_call start - # tag (guided decoding can split it across deltas). - start_idx = cur_text.find(self.tool_call_start_token) - if start_idx == -1 and self._streaming_tool_name is None: - keep = partial_tag_overlap(cur_text, self.tool_call_start_token) - self._buffer = cur_text[len(cur_text) - keep :] if keep else "" - break - - # === Phase 1: Detect tool name (send when first arg_key is seen) = - # ``name_new`` mirrors the old ``name_delta is not None``: it marks - # the chunk that first emits this tool's name, which is the only - # chunk the wrapper mints an id + type for. - name_new = False - pending_name: str | None = None - if self._streaming_tool_name is None: - arg_idx = cur_text.find(self.arg_key_start_token, start_idx) - end_idx = cur_text.find(self.tool_call_end_token, start_idx) - if arg_idx == -1 and end_idx == -1: - # tool name not yet complete; keep buffering from - # tool_call_start - self._buffer = cur_text[start_idx:] - break - - name_start = start_idx + len(self.tool_call_start_token) - name_end = arg_idx if arg_idx != -1 else end_idx - - tool_name = cur_text[name_start:name_end].strip() - self._streaming_tool_name = tool_name - - if arg_idx != -1: - self._buffer = cur_text[arg_idx:] - else: - self._buffer = cur_text[end_idx:] - - # Increment tool_id and mark that a name chunk should be emitted - self.current_tool_id += 1 - name_new = True - pending_name = tool_name - - # Check if buffer already has complete arguments - # (all-in-one-delta); otherwise only the name is ready now. - if self.tool_call_end_token not in self._buffer: - tool_calls.append( - StreamToolCall( - index=self.current_tool_id, - new=True, - name=tool_name, - arguments=None, - ) - ) - break - # Buffer already has a complete tool call; fall through to - # phase 2 below. - - # === Phase 2: Incremental argument streaming === - result = self._extract_streaming_incremental(name_new, pending_name, tools) - if result: - tool_calls.extend(result["tool_calls"]) - - # Continue only after the current tool call has completed and the - # remaining structural payload starts with another tool call. - # A literal ```` may legitimately appear inside an open - # string argument; in that case the current tool name is still set - # and the buffer has not been consumed, so continuing would spin on - # the same buffer forever. - if self._streaming_tool_name is None and self._buffer.startswith( - self.tool_call_start_token - ): - continue - break - - if content_delta is not None or tool_calls: - return StreamDelta(content=content_delta, tool_calls=tool_calls) - return None - - def _make_args_delta(self, argument_diff: str) -> StreamDelta: - """Build an args-only streaming delta (no id/type -> new=False).""" - return StreamDelta( - content=None, - tool_calls=[ - StreamToolCall( - index=self.current_tool_id, - new=False, - name=None, - arguments=argument_diff, - ) - ], - ) - - def _extract_streaming_incremental( - self, - name_new: bool, - pending_name: str | None, - tools: list[ToolSchema] | None, - ) -> StreamDelta | None: - """Incremental phase-2: scan tags in buffer, emit JSON diffs. - - Strategy: - - Track completed args and emit each one as a JSON fragment. - - For string-typed args, stream the value character-by-character. - - Withhold the closing ``}`` until ```` is seen. - - We build JSON manually via fragments rather than using json.dumps - with a cursor, because json.dumps of partial-vs-full string values - produces incompatible prefixes (e.g. ``""}`` vs ``"Hello"}``). - """ - buf = self._buffer - is_complete = self.tool_call_end_token in buf - - if is_complete: - end_idx = buf.find(self.tool_call_end_token) - args_text = buf[:end_idx] - remaining = buf[end_idx + len(self.tool_call_end_token) :] - else: - args_text = buf - remaining = "" - - # --- scan all fully closed kv pairs --- - arg_pairs = self.func_args_regex.findall(args_text) - for key, value in arg_pairs: - key = key.strip() - if key not in self._completed_args: - self._completed_args[key] = _parse_value( - value, self._streaming_tool_name or "", key, tools - ) - - # --- detect partial (unclosed) kv at the tail --- - last_closed_end = 0 - for m in self.func_args_regex.finditer(args_text): - last_closed_end = m.end() - tail = args_text[last_closed_end:] - - partial_key: str | None = None - partial_value: str | None = None - - ak_start = tail.find(self.arg_key_start_token) - if ak_start != -1: - ak_end = tail.find( - self.arg_key_end_token, - ak_start + len(self.arg_key_start_token), - ) - if ak_end != -1: - partial_key = tail[ - ak_start + len(self.arg_key_start_token) : ak_end - ].strip() - self._current_arg_key = partial_key - self._current_arg_is_string = _is_only_string_type( - self._streaming_tool_name or "", - partial_key, - tools, - ) - - av_start = tail.find(self.arg_value_start_token, ak_end) - if av_start != -1: - val_content_start = av_start + len(self.arg_value_start_token) - if self._current_arg_is_string: - raw_value = tail[val_content_start:] - # Guided decoding can split ```` across - # deltas; hold back a partial closing tag so it is not - # streamed as part of the value. - keep = len(raw_value) - partial_tag_overlap( - raw_value, self.arg_value_end_token - ) - partial_value = raw_value[:keep] - else: - # key not yet closed - self._current_arg_key = None - self._current_arg_is_string = False - - # --- build the current JSON snapshot as a string --- - # We construct JSON manually so we can precisely control - # what gets sent incrementally. - snapshot_parts: list[str] = [] - for k, v in self._completed_args.items(): - k_json = json.dumps(k, ensure_ascii=False) - v_json = json.dumps(v, ensure_ascii=False) - snapshot_parts.append(f"{k_json}: {v_json}") - - if partial_key is not None and partial_value is not None: - k_json = json.dumps(partial_key, ensure_ascii=False) - # For string partial value, we build the JSON string - # WITHOUT the closing quote, so the prefix stays stable - # as the value grows. The closing `"` and `}` will be - # sent when the value or tool_call closes. - escaped_val = json.dumps(partial_value, ensure_ascii=False)[1:-1] - # Note: no closing " here - it's appended only on close - snapshot_parts.append(f'{k_json}: "{escaped_val}') - - snapshot = "{" + ", ".join(snapshot_parts) + "}" - - # --- compute diff --- - argument_diff: str | None = None - - if is_complete: - # Tool call finished - send everything remaining. - # Build final snapshot with proper JSON (all values closed). - final_args = dict(self._completed_args) - final_json = json.dumps(final_args, ensure_ascii=False) - if self._streamed_json_len < len(final_json): - argument_diff = final_json[self._streamed_json_len :] - self._streamed_json_len = len(final_json) - - # Record into prev_tool_call_arr - self.prev_tool_call_arr.append( - { - "name": self._streaming_tool_name, - "arguments": final_args, - } - ) - self.streamed_args_for_tool.append(final_json) - - self._reset_streaming_tool_state() - self._buffer = remaining - else: - # Still in progress - withhold the tail. - # For open strings: snapshot ends with ...partial_val} - # we withhold "}" (1 char) - the missing closing " will - # be sent when the value closes. - # For no open string: snapshot ends with ...value"} - # we withhold "}" (1 char). - end = len(snapshot) - 1 # exclude trailing "}" - if end > self._streamed_json_len: - argument_diff = snapshot[self._streamed_json_len : end] - self._streamed_json_len = end - - # --- construct return dict --- - if name_new and argument_diff: - return StreamDelta( - content=None, - tool_calls=[ - StreamToolCall( - index=self.current_tool_id, - new=True, - name=pending_name, - arguments=argument_diff, - ) - ], - ) - elif name_new: - return StreamDelta( - content=None, - tool_calls=[ - StreamToolCall( - index=self.current_tool_id, - new=True, - name=pending_name, - arguments=None, - ) - ], - ) - elif argument_diff: - return self._make_args_delta(argument_diff) - else: - return None - - -def build_tool_extractor( - tokenizer: TokenizerLike, *, strict: bool -) -> HYV4ToolExtractor: - return HYV4ToolExtractor( - tokenizer.get_vocab(), detect_token_suffix(tokenizer), strict - ) - - -def _tools_to_plain(tools) -> list[ToolSchema] | None: - """Convert tool-schema objects into plain dicts for the extractor.""" - if not tools: - return None - return [ - ToolSchema(name=t.function.name, parameters=t.function.parameters) - for t in tools - ] - - -class HYV4ToolParser(ToolParser): - """vLLM adapter around :class:`HYV4ToolExtractor`. - - Converts the extractor's plain dicts to vLLM protocol objects, mints - tool-call ids, and mirrors ``prev_tool_call_arr`` / - ``streamed_args_for_tool`` so the serving layer sees live state. - """ - - # HYV4 emits its own /// - # structural tokens, not a plain JSON array. The generic required/named - # JSON streaming path (streaming.extract_required_tool_call_streaming) - # cannot parse this format and produces misaligned multi-tool deltas in - # streaming mode. Fall back to our own (auto-path) parser for - # required/named too, so streaming and non-streaming stay consistent. - supports_required_and_named: bool = False - structural_tag_model: str = "hy_v4" - - def __init__(self, tokenizer: TokenizerLike, tools=None): - super().__init__(tokenizer, tools) - - # Tools are supplied at construction (``tool_parser(tokenizer, - # request.tools)``); keep a plain-dict copy for the extractor. - self._plain_tools = _tools_to_plain(tools) - # Malformed tag structure is rejected and returned as raw content - # instead of a half-parsed tool call. - self._extractor = build_tool_extractor(tokenizer, strict=True) - self._mirror_state() - - def get_structural_tag( - self, - request: ChatCompletionRequest | ResponsesRequest, - *, - reasoning: bool = False, - strict_level: ToolStrictLevel = ToolStrictLevel.AUTO, - ) -> StructuralTag | None: - """Build a structural tag matching HYV4's tool tokens. - - Overridden only to pass the per-checkpoint ``token_suffix`` through to - the builder; everything else follows the base implementation. - - Named (forced) tool choice is constrained with the same structural tag: - because ``supports_required_and_named`` is False, required *and* named - requests are parsed by the HYV4 extractor, which only recognizes the HYV4 - structural tokens -- the default JSON guided-decoding path would make - the model emit JSON the extractor silently drops. - - Args: - request: The request being adjusted. - reasoning: Whether the grammar also covers the reasoning phase. - strict_level: Server-side floor from ``--tool-strict-level``. - - Returns: - The structural tag, or None when structural tagging does not apply. - - """ - if not envs.VLLM_ENFORCE_STRICT_TOOL_CALLING: - return None - - # Only constrain required / forced tool choice with the structural tag. - # For "auto", tool calling is optional and most requests never emit a - # tool call, so applying the grammar to every auto request would make - # the whole stream pay the output inflation and the per-token - # bitmask/accept overhead for a minority benefit. Skip it here so auto - # falls through to free generation; the HYV4 parser still reads back - # any tool calls the model emits. - tool_choice = request.tool_choice - if ( - tool_choice is None - or tool_choice == "auto" - or getattr(tool_choice, "mode", None) == "auto" - ): - return None - - from vllm.tool_parsers.structural_tag_registry import get_model_structural_tag - - try: - return get_model_structural_tag( - model=self.structural_tag_model, - tools=request.tools, - tool_choice=request.tool_choice, - reasoning=reasoning, - token_suffix=self._extractor.token_suffix, - strict_level=strict_level, - ) - except Exception: - logger.warning( - "HYV4ToolParser failed to build a structural tag; falling back " - "to the default guided-decoding path.", - exc_info=True, - ) - return None - - def _mirror_state(self) -> None: - # Alias the SAME list objects the extractor appends to, so the serving - # layer reads live state. ``current_tool_id`` is an int, so re-read it. - self.prev_tool_call_arr = self._extractor.prev_tool_call_arr - self.streamed_args_for_tool = self._extractor.streamed_args_for_tool - self.current_tool_id = self._extractor.current_tool_id - - @staticmethod - def _is_guided(request: ChatCompletionRequest) -> bool: - """Whether a structural tag constrains this request's decoding. - - ``get_structural_tag`` only applies to required / named tool choice and - the serving layer records the result on the request. A structural tag - constrains strings rather than token ids, so the ```` marker - can arrive split across sub-word tokens and must be detected on the - string level. - - Args: - request: The request being streamed. - - Returns: - True when the streaming parser must use the string-marker path. - - """ - structured_outputs = getattr(request, "structured_outputs", None) - return ( - structured_outputs is not None - and structured_outputs.structural_tag is not None - ) - - def _tools(self, request: ChatCompletionRequest) -> list[ToolSchema] | None: - # Prefer the tools passed at construction; fall back to the request. - if self._plain_tools is not None: - return self._plain_tools - return _tools_to_plain(getattr(request, "tools", None)) - - def extract_tool_calls( - self, - model_output: str, - request: ChatCompletionRequest, - ) -> ExtractedToolCallInformation: - result = self._extractor.extract_tool_calls(model_output, self._tools(request)) - self._mirror_state() - return ExtractedToolCallInformation( - tools_called=result["tools_called"], - content=result["content"], - tool_calls=[ - ToolCall( - type="function", - function=FunctionCall( - name=tc["name"], - arguments=tc["arguments"], - ), - ) - for tc in result["tool_calls"] - ], - ) - - def extract_tool_calls_streaming( - self, - previous_text: str, - current_text: str, - delta_text: str, - previous_token_ids: Sequence[int], - current_token_ids: Sequence[int], - delta_token_ids: Sequence[int], - request: ChatCompletionRequest, - ) -> DeltaMessage | None: - delta = self._extractor.extract_tool_calls_streaming( - previous_text, - current_text, - delta_text, - previous_token_ids, - current_token_ids, - delta_token_ids, - self._tools(request), - guided=self._is_guided(request), - ) - self._mirror_state() - if delta is None: - return None - - tool_calls: list[DeltaToolCall] = [] - for tc in delta.get("tool_calls", []): - # Only set the fields that are present, leaving the rest UNSET (not - # None). Streaming serialization uses exclude_unset, so an explicit - # ``arguments=None`` would emit ``"arguments": null`` and break - # clients that concatenate argument deltas. Matches hy_v3. - fn_kwargs: dict[str, str] = {} - name = tc.get("name") - if name is not None: - fn_kwargs["name"] = name - arguments = tc.get("arguments") - if arguments is not None: - fn_kwargs["arguments"] = arguments - function = DeltaFunctionCall(**fn_kwargs) - if tc.get("new"): - # First chunk for this tool: mint an id + set type here only. - tool_calls.append( - DeltaToolCall( - index=tc["index"], - id=make_tool_call_id(), - type="function", - function=function, - ) - ) - else: - tool_calls.append(DeltaToolCall(index=tc["index"], function=function)) - - # A single engine delta can carry content plus N drained tool calls. - # Only set the fields that are present so streaming serialization - # (exclude_unset) never emits ``content: null`` on a tool-call chunk. - content = delta.get("content") - kwargs: dict[str, Any] = {} - if content is not None: - kwargs["content"] = content - if tool_calls: - kwargs["tool_calls"] = tool_calls - if not kwargs: - return None - return DeltaMessage(**kwargs) diff --git a/vllm/tool_parsers/inkling_tool_parser.py b/vllm/tool_parsers/inkling_tool_parser.py deleted file mode 100644 index ab760015af..0000000000 --- a/vllm/tool_parsers/inkling_tool_parser.py +++ /dev/null @@ -1,11 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from vllm.parser.engine.registered_adapters import InklingParserToolAdapter - - -class InklingEngineToolParser(InklingParserToolAdapter): # type: ignore[valid-type, misc] - # No Inkling structural-tag grammar is wired up yet; fall back to auto - # parsing for named/required tool choice. - structural_tag_model = None - supports_required_and_named = False diff --git a/vllm/tool_parsers/internlm2_tool_parser.py b/vllm/tool_parsers/internlm2_tool_parser.py deleted file mode 100644 index 9bec016ec9..0000000000 --- a/vllm/tool_parsers/internlm2_tool_parser.py +++ /dev/null @@ -1,248 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -import json -from collections.abc import Sequence - -import partial_json_parser -from partial_json_parser.core.options import Allow - -from vllm.entrypoints.generate.base.protocol import ( - DeltaFunctionCall, - DeltaMessage, - DeltaToolCall, - ExtractedToolCallInformation, - FunctionCall, - ToolCall, -) -from vllm.entrypoints.openai.chat_completion.protocol import ( - ChatCompletionRequest, -) -from vllm.entrypoints.openai.responses.protocol import ResponsesRequest -from vllm.logger import init_logger -from vllm.renderers.chat_utils import make_tool_call_id -from vllm.tokenizers import TokenizerLike -from vllm.tool_parsers.abstract_tool_parser import ( - Tool, - ToolParser, -) -from vllm.tool_parsers.utils import find_common_prefix, is_complete_json - -logger = init_logger(__name__) - - -class Internlm2ToolParser(ToolParser): - def __init__(self, tokenizer: TokenizerLike, tools: list[Tool] | None = None): - super().__init__(tokenizer, tools) - self.position = 0 - - def adjust_request( - self, request: ChatCompletionRequest | ResponsesRequest - ) -> ChatCompletionRequest | ResponsesRequest: - request = super().adjust_request(request) - if request.tools and request.tool_choice != "none": - # do not skip special tokens because internlm use the special - # tokens to indicate the start and end of the tool calls - # information. - request.skip_special_tokens = False - return request - - def get_arguments(self, obj): - if "parameters" in obj: - return obj.get("parameters") - elif "arguments" in obj: - return obj.get("arguments") - return None - - def extract_tool_calls_streaming( - self, - previous_text: str, - current_text: str, - delta_text: str, - previous_token_ids: Sequence[int], - current_token_ids: Sequence[int], - delta_token_ids: Sequence[int], - request: ChatCompletionRequest, - ) -> DeltaMessage | None: - if "<|action_start|>" not in current_text: - self.position = len(current_text) - return DeltaMessage(content=delta_text) - # if the tool call is sent, return an empty delta message - # to make sure the finish_reason will be sent correctly. - if self.current_tool_id > 0: - return DeltaMessage(content="") - - last_pos = self.position - if "<|action_start|><|plugin|>" not in current_text[last_pos:]: - return None - - new_delta = current_text[last_pos:] - text, action = new_delta.split("<|action_start|><|plugin|>") - - if len(text) > 0: - self.position = self.position + len(text) - return DeltaMessage(content=text) - - action = action.strip() - action = action.split("<|action_end|>".strip())[0] - - # bit mask flags for partial JSON parsing. If the name hasn't been - # sent yet, don't allow sending - # an incomplete string since OpenAI only ever (as far as I have - # seen) allows sending the entire tool/ function name at once. - flags = Allow.ALL if self.current_tool_name_sent else Allow.ALL & ~Allow.STR - - try: - parsable_arr = action - - # tool calls are generated in an object in internlm2 - # it's not support parallel tool calls - try: - tool_call_arr: dict = partial_json_parser.loads(parsable_arr, flags) - except partial_json_parser.core.exceptions.MalformedJSON: - logger.debug("not enough tokens to parse into JSON yet") - return None - - # if the current tool name hasn't been sent, send if available - # - otherwise send nothing - if not self.current_tool_name_sent: - function_name = tool_call_arr.get("name") - if function_name: - self.current_tool_id = self.current_tool_id + 1 - delta = DeltaMessage( - tool_calls=[ - DeltaToolCall( - index=self.current_tool_id, - type="function", - id=make_tool_call_id(), - function=DeltaFunctionCall( - name=function_name - ).model_dump(exclude_none=True), - ) - ] - ) - self.current_tool_name_sent = True - self.streamed_args_for_tool.append("") - else: - delta = None - # now we know we're on the same tool call and we're streaming - # arguments - else: - prev_arguments = self.get_arguments( - self.prev_tool_call_arr[self.current_tool_id] - ) - cur_arguments = self.get_arguments(tool_call_arr) - - # not arguments generated - if not cur_arguments and not prev_arguments: - delta = None - # will never happen - elif not cur_arguments and prev_arguments: - logger.error( - "INVARIANT - impossible to have arguments reset mid-arguments" - ) - delta = None - # first time to get parameters - elif cur_arguments and not prev_arguments: - cur_arguments_json = json.dumps(cur_arguments, ensure_ascii=False) - - match_start = cur_arguments_json.find(delta_text) - if match_start != -1: - arguments_delta = cur_arguments_json[ - : match_start + len(delta_text) - ] - elif is_complete_json(parsable_arr): - # Complete in this delta: send whole, don't drop. - arguments_delta = cur_arguments_json - else: - # Still partial: wait for more text. - return None - delta = DeltaMessage( - tool_calls=[ - DeltaToolCall( - index=self.current_tool_id, - function=DeltaFunctionCall( - arguments=arguments_delta - ).model_dump(exclude_none=True), - ) - ] - ) - self.streamed_args_for_tool[self.current_tool_id] += arguments_delta - # both prev and cur parameters, send the increase parameters - elif cur_arguments and prev_arguments: - cur_args_json = json.dumps(cur_arguments, ensure_ascii=False) - prev_args_json = json.dumps(prev_arguments, ensure_ascii=False) - sent = len(self.streamed_args_for_tool[self.current_tool_id]) - - # Partial JSON is serialized with synthetic closers, so - # only the part shared with the previous parse is final. - if is_complete_json(parsable_arr): - argument_diff = cur_args_json[sent:] - elif cur_args_json != prev_args_json: - prefix = find_common_prefix(prev_args_json, cur_args_json) - argument_diff = prefix[sent:] - else: - argument_diff = "" - - delta = DeltaMessage( - tool_calls=[ - DeltaToolCall( - index=self.current_tool_id, - function=DeltaFunctionCall( - arguments=argument_diff - ).model_dump(exclude_none=True), - ) - ] - ) - self.streamed_args_for_tool[self.current_tool_id] += argument_diff - - # check to see if the name is defined and has been sent. if so, - # stream the name - otherwise keep waiting - # finish by setting old and returning None as base case - tool_call_arr["arguments"] = self.get_arguments(tool_call_arr) - self.prev_tool_call_arr = [tool_call_arr] - return delta - except Exception: - logger.exception("Error trying to handle streaming tool call.") - logger.debug( - "Skipping chunk as a result of tool streaming extraction error" - ) - return None - - def extract_tool_calls( - self, - model_output: str, - request: ChatCompletionRequest, - ) -> ExtractedToolCallInformation: - text = model_output - tools = self.tools - if "<|action_start|><|plugin|>" in text: - text, action = text.split("<|action_start|><|plugin|>") - action = action.split("<|action_end|>".strip())[0] - action = action[action.find("{") :] - action_dict = json.loads(action) - name, parameters = ( - action_dict["name"], - json.dumps( - action_dict.get("parameters", action_dict.get("arguments", {})), - ensure_ascii=False, - ), - ) - - if not tools or name not in [t.function.name for t in tools]: - ExtractedToolCallInformation( - tools_called=False, tool_calls=[], content=text - ) - - tool_calls = [ - ToolCall(function=FunctionCall(name=name, arguments=parameters)) - ] - return ExtractedToolCallInformation( - tools_called=True, - tool_calls=tool_calls, - content=text if len(text) > 0 else None, - ) - - return ExtractedToolCallInformation( - tools_called=False, tool_calls=[], content=text - ) diff --git a/vllm/tool_parsers/jamba_tool_parser.py b/vllm/tool_parsers/jamba_tool_parser.py deleted file mode 100644 index 83eea1d580..0000000000 --- a/vllm/tool_parsers/jamba_tool_parser.py +++ /dev/null @@ -1,337 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -import json -from collections.abc import Sequence - -import partial_json_parser -import regex as re -from partial_json_parser.core.options import Allow - -from vllm.entrypoints.generate.base.protocol import ( - DeltaFunctionCall, - DeltaMessage, - DeltaToolCall, - ExtractedToolCallInformation, - FunctionCall, - ToolCall, -) -from vllm.entrypoints.openai.chat_completion.protocol import ( - ChatCompletionRequest, -) -from vllm.entrypoints.openai.responses.protocol import ResponsesRequest -from vllm.logger import init_logger -from vllm.renderers.chat_utils import make_tool_call_id -from vllm.tokenizers import TokenizerLike -from vllm.tool_parsers.abstract_tool_parser import Tool, ToolParser -from vllm.tool_parsers.utils import extract_intermediate_diff, is_complete_json -from vllm.utils.mistral import is_mistral_tokenizer - -logger = init_logger(__name__) - - -class JambaToolParser(ToolParser): - def __init__(self, tokenizer: TokenizerLike, tools: list[Tool] | None = None): - super().__init__(tokenizer, tools) - - if is_mistral_tokenizer(self.model_tokenizer): - raise ValueError( - "Detected a MistralTokenizer tokenizer when using a Jamba model" - ) - - self.current_tool_name_sent: bool = False - self.prev_tool_call_arr: list[dict] = [] - self.current_tool_id: int = -1 - self.streamed_args_for_tool: list[ - str - ] = [] # map what has been streamed for each tool so far to a list - - self.tool_calls_start_token: str = "" - self.tool_calls_end_token: str = "" - - self.tool_calls_regex = re.compile( - rf"{self.tool_calls_start_token}(.*?){self.tool_calls_end_token}", re.DOTALL - ) - - if not self.model_tokenizer: - raise ValueError( - "The model tokenizer must be passed to the ToolParser " - "constructor during construction." - ) - self.tool_calls_start_token_id = self.vocab.get(self.tool_calls_start_token) - self.tool_calls_end_token_id = self.vocab.get(self.tool_calls_end_token) - if ( - self.tool_calls_start_token_id is None - or self.tool_calls_end_token_id is None - ): - raise RuntimeError( - "Jamba Tool parser could not locate tool calls start/end " - "tokens in the tokenizer!" - ) - - def adjust_request( - self, request: ChatCompletionRequest | ResponsesRequest - ) -> ChatCompletionRequest | ResponsesRequest: - request = super().adjust_request(request) - if request.tools and request.tool_choice != "none": - # do not skip special tokens because jamba use the special - # tokens to indicate the start and end of the tool calls - # information. - request.skip_special_tokens = False - return request - - def extract_tool_calls( - self, model_output: str, request: ChatCompletionRequest - ) -> ExtractedToolCallInformation: - # sanity check; avoid unnecessary processing - if self.tool_calls_start_token not in model_output: - return ExtractedToolCallInformation( - tools_called=False, tool_calls=[], content=model_output - ) - - else: - try: - # use a regex to find the tool call between the tags - function_calls = self.tool_calls_regex.findall(model_output)[0] - - # load the JSON, and then use it to build the Function and - # Tool Call - raw_function_calls = json.loads(function_calls) - tool_calls = [ - ToolCall( - type="function", - function=FunctionCall( - name=function_call["name"], - # function call args are JSON but as a string - arguments=json.dumps( - function_call["arguments"], ensure_ascii=False - ), - ), - ) - for function_call in raw_function_calls - ] - - content = model_output[: model_output.find(self.tool_calls_start_token)] - return ExtractedToolCallInformation( - tools_called=True, - tool_calls=tool_calls, - content=content if (len(content) > 0 and content != " ") else None, - ) - - except Exception: - logger.exception("Error in extracting tool call from response.") - return ExtractedToolCallInformation( - tools_called=False, tool_calls=[], content=model_output - ) - - def extract_tool_calls_streaming( - self, - previous_text: str, - current_text: str, - delta_text: str, - previous_token_ids: Sequence[int], - current_token_ids: Sequence[int], - delta_token_ids: Sequence[int], - request: ChatCompletionRequest, - ) -> DeltaMessage | None: - # if the tool call token is not in the tokens generated so far, append - # output to contents since it's not a tool - if self.tool_calls_start_token not in current_text: - return DeltaMessage(content=delta_text) - - # if the tool call token ID IS in the tokens generated so far, that - # means we're parsing as tool calls now - - # handle if we detected the start of tool calls token which means - # the start of tool calling - if ( - self.tool_calls_start_token_id in delta_token_ids - and len(delta_token_ids) == 1 - ): - # if it's the only token, return None, so we don't send a chat - # completion and don't send a control token - return None - - # bit mask flags for partial JSON parsing. If the name hasn't been - # sent yet, don't allow sending - # an incomplete string since OpenAI only ever (as far as I have - # seen) allows sending the entire tool/ function name at once. - flags = Allow.ALL if self.current_tool_name_sent else Allow.ALL & ~Allow.STR - try: - # Extract the tool calls between the special tool call tokens - parsable_arr = current_text.split(self.tool_calls_start_token)[-1].split( - self.tool_calls_end_token - )[0] - - # tool calls are generated in an array, so do partial JSON - # parsing on the entire array - try: - tool_call_arr: list[dict] = partial_json_parser.loads( - parsable_arr, flags - ) - except partial_json_parser.core.exceptions.MalformedJSON: - logger.debug("not enough tokens to parse into JSON yet") - return None - - # select as the current tool call the one we're on the state at - - current_tool_call: dict = ( - tool_call_arr[self.current_tool_id] if len(tool_call_arr) > 0 else {} - ) - - # case -- if no tokens have been streamed for the tool, e.g. - # only the array brackets, stream nothing - if len(tool_call_arr) == 0: - return None - - # case: we are starting a new tool in the array - # -> array has > 0 length AND length has moved past cursor - elif ( - len(tool_call_arr) > 0 and len(tool_call_arr) > self.current_tool_id + 1 - ): - # if we're moving on to a new call, first make sure we - # haven't missed anything in the previous one that was - # auto-generated due to JSON completions, but wasn't - # streamed to the client yet. - if self.current_tool_id >= 0: - diff: str | None = current_tool_call.get("arguments") - - if diff: - diff = json.dumps(diff, ensure_ascii=False).replace( - self.streamed_args_for_tool[self.current_tool_id], "" - ) - delta = DeltaMessage( - tool_calls=[ - DeltaToolCall( - index=self.current_tool_id, - function=DeltaFunctionCall( - arguments=diff - ).model_dump(exclude_none=True), - ) - ] - ) - self.streamed_args_for_tool[self.current_tool_id] += diff - else: - delta = None - else: - delta = None - # re-set stuff pertaining to progress in the current tool - self.current_tool_id = len(tool_call_arr) - 1 - self.current_tool_name_sent = False - self.streamed_args_for_tool.append("") - logger.debug("starting on new tool %d", self.current_tool_id) - return delta - - # case: update an existing tool - this is handled below - - # if the current tool name hasn't been sent, send if available - # - otherwise send nothing - if not self.current_tool_name_sent: - function_name = current_tool_call.get("name") - if function_name: - delta = DeltaMessage( - tool_calls=[ - DeltaToolCall( - index=self.current_tool_id, - type="function", - id=make_tool_call_id(), - function=DeltaFunctionCall( - name=function_name - ).model_dump(exclude_none=True), - ) - ] - ) - self.current_tool_name_sent = True - else: - delta = None - - # now we know we're on the same tool call and we're streaming - # arguments - else: - prev_arguments = self.prev_tool_call_arr[self.current_tool_id].get( - "arguments" - ) - cur_arguments = current_tool_call.get("arguments") - - new_text = delta_text.replace("'", '"') - - if not cur_arguments and not prev_arguments: - delta = None - elif not cur_arguments and prev_arguments: - logger.error( - "INVARIANT - impossible to have arguments reset mid-arguments" - ) - delta = None - elif cur_arguments and not prev_arguments: - cur_arguments_json = json.dumps(cur_arguments, ensure_ascii=False) - logger.debug("finding %s in %s", new_text, cur_arguments_json) - - # `new_text` may not appear verbatim in the re-serialized JSON. - match_start = cur_arguments_json.find(new_text) - if match_start != -1: - arguments_delta = cur_arguments_json[ - : match_start + len(new_text) - ] - elif is_complete_json(parsable_arr): - # Complete in this delta: send whole, don't drop. - arguments_delta = cur_arguments_json - else: - # Still partial: wait for more text. - return None - logger.debug( - "First tokens in arguments received: %s", arguments_delta - ) - delta = DeltaMessage( - tool_calls=[ - DeltaToolCall( - index=self.current_tool_id, - function=DeltaFunctionCall( - arguments=arguments_delta - ).model_dump(exclude_none=True), - ) - ] - ) - self.streamed_args_for_tool[self.current_tool_id] += arguments_delta - - elif cur_arguments and prev_arguments: - cur_args_json = json.dumps(cur_arguments, ensure_ascii=False) - prev_args_json = json.dumps(prev_arguments, ensure_ascii=False) - logger.debug( - "Searching for diff between \n%s\n%s", - cur_args_json, - prev_args_json, - ) - - argument_diff = extract_intermediate_diff( - cur_args_json, prev_args_json - ) - logger.debug("got arguments diff: %s", argument_diff) - delta = DeltaMessage( - tool_calls=[ - DeltaToolCall( - index=self.current_tool_id, - function=DeltaFunctionCall( - arguments=argument_diff - ).model_dump(exclude_none=True), - ) - ] - ) - self.streamed_args_for_tool[self.current_tool_id] += argument_diff - else: - # try parsing it with regular JSON - if it works we're - # at the end, and we need to send the difference between - # tokens streamed so far and the valid JSON - delta = None - - # check to see if the name is defined and has been sent. if so, - # stream the name - otherwise keep waiting - # finish by setting old and returning None as base case - self.prev_tool_call_arr = tool_call_arr - return delta - - except Exception: - logger.exception("Error trying to handle streaming tool call.") - logger.debug( - "Skipping chunk as a result of tool streaming extraction error" - ) - return None diff --git a/vllm/tool_parsers/k2_horizon_tool_parser.py b/vllm/tool_parsers/k2_horizon_tool_parser.py deleted file mode 100644 index a63f5fa122..0000000000 --- a/vllm/tool_parsers/k2_horizon_tool_parser.py +++ /dev/null @@ -1,462 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -import json -from collections.abc import Sequence -from typing import Any - -import regex as re -from openai.types.responses import ToolChoiceCustom, ToolChoiceFunction - -from vllm.entrypoints.generate.base.protocol import ( - DeltaFunctionCall, - DeltaMessage, - DeltaToolCall, - ExtractedToolCallInformation, - FunctionCall, - ToolCall, -) -from vllm.entrypoints.openai.chat_completion.protocol import ( - ChatCompletionNamedToolChoiceParam, - ChatCompletionRequest, -) -from vllm.entrypoints.openai.responses.protocol import ResponsesRequest -from vllm.logger import init_logger -from vllm.tokenizers import TokenizerLike -from vllm.tool_parsers.abstract_tool_parser import Tool, ToolParser -from vllm.tool_parsers.utils import ( - coerce_to_schema_type, - extract_types_from_schema, - find_tool_name, - find_tool_properties, - partial_tag_overlap, -) - -logger = init_logger(__name__) - -ToolParserRequest = ChatCompletionRequest | ResponsesRequest - - -class K2HorizonToolParser(ToolParser): - """Parser for K2 Horizon's IFM tool-call formats.""" - - supports_required_and_named = False - engine_based_streaming = True - - _SUPPORTED_TOOL_FORMATS = frozenset({"json", "xml", "xml_typed"}) - - _TOOL_CALLS_START = "" - _TOOL_CALLS_END = "" - _TOOL_CALL_START = "" - _TOOL_CALL_END = "" - _ARG_KEY_START = "" - _ARG_KEY_END = "" - _ARG_TYPE_START = "" - _ARG_TYPE_END = "" - _ARG_VALUE_START = "" - _ARG_VALUE_END = "" - - _TOOL_CALL_RE = re.compile( - re.escape(_TOOL_CALL_START) + r"(.*?)" + re.escape(_TOOL_CALL_END), - re.DOTALL, - ) - _ARG_RE = re.compile( - re.escape(_ARG_KEY_START) - + r"(.*?)" - + re.escape(_ARG_KEY_END) - + r"\s*(?:" - + re.escape(_ARG_TYPE_START) - + r"(.*?)" - + re.escape(_ARG_TYPE_END) - + r"\s*)?" - + re.escape(_ARG_VALUE_START) - + r"(.*?)" - + re.escape(_ARG_VALUE_END), - re.DOTALL, - ) - _ARG_MARKERS = ( - _ARG_KEY_START, - _ARG_KEY_END, - _ARG_TYPE_START, - _ARG_TYPE_END, - _ARG_VALUE_START, - _ARG_VALUE_END, - ) - - def __init__( - self, - tokenizer: TokenizerLike, - tools: list[Tool] | None = None, - ) -> None: - super().__init__(tokenizer, tools) - self.tool_format = "xml" - self._stream_text = "" - self._stream_cursor = 0 - self._stream_group_start: int | None = None - self._stream_resolved = False - self._pending_content = "" - self._content_seen = False - - @classmethod - def _request_tool_format(cls, request: ToolParserRequest) -> str: - chat_template_kwargs = request.chat_template_kwargs or {} - tool_format = chat_template_kwargs.get("tool_call_format", "xml") - if ( - not isinstance(tool_format, str) - or tool_format not in cls._SUPPORTED_TOOL_FORMATS - ): - supported = ", ".join(sorted(cls._SUPPORTED_TOOL_FORMATS)) - raise ValueError( - f"Unsupported tool_call_format {tool_format!r}. " - f"Supported values: {supported}." - ) - return tool_format - - def adjust_request(self, request: ToolParserRequest) -> ToolParserRequest: - self._request_tool_format(request) - return request - - @staticmethod - def _content_or_none(content: str) -> str | None: - return content if content.strip() else None - - @classmethod - def _no_tool_calls(cls, content: str) -> ExtractedToolCallInformation: - return ExtractedToolCallInformation( - tools_called=False, - tool_calls=[], - content=cls._content_or_none(content), - ) - - @staticmethod - def _make_tool_call(name: str, arguments: dict[str, Any]) -> ToolCall: - return ToolCall( - type="function", - function=FunctionCall( - name=name, - arguments=json.dumps( - arguments, - ensure_ascii=False, - allow_nan=False, - ), - ), - ) - - @staticmethod - def _base_arg_type(arg_type: str | None) -> str | None: - if not arg_type: - return None - return arg_type.split("[", 1)[0].strip().lower() - - @staticmethod - def _coerce_json_value(value: Any, schema: Any) -> Any: - if not isinstance(schema, dict): - return value - raw_value = ( - value - if isinstance(value, str) - else json.dumps(value, ensure_ascii=False, allow_nan=False) - ) - return coerce_to_schema_type(raw_value, extract_types_from_schema(schema)) - - @classmethod - def _coerce_xml_value( - cls, - value: str, - schema: Any, - explicit_type: str | None, - ) -> Any: - explicit_base = cls._base_arg_type(explicit_type) - target_types = extract_types_from_schema(schema) - if isinstance(schema, dict): - if explicit_base in target_types and len(set(target_types)) > 1: - target_types = [explicit_base] - elif explicit_base == "any": - return value - elif explicit_base: - target_types = [explicit_base] - - raw_value = value if "string" in target_types else value.strip() - return coerce_to_schema_type(raw_value, target_types) - - @staticmethod - def _named_tool(request: ToolParserRequest) -> str | None: - tool_choice = request.tool_choice - if isinstance(tool_choice, ChatCompletionNamedToolChoiceParam): - return tool_choice.function.name - if isinstance(tool_choice, (ToolChoiceFunction, ToolChoiceCustom)): - return tool_choice.name - return None - - @classmethod - def _validate_tool_name( - cls, - name: Any, - request: ToolParserRequest, - ) -> str: - if ( - not isinstance(name, str) - or not name - or any(char.isspace() for char in name) - ): - raise ValueError("IFM tool call has an invalid function name.") - if request.tools and not find_tool_name(request.tools, name): - raise ValueError(f"Unknown IFM tool name: {name}.") - named_tool = cls._named_tool(request) - if named_tool is not None and name != named_tool: - raise ValueError(f"Unexpected IFM tool name: {name}.") - return name - - @classmethod - def _parse_json_call( - cls, - body: str, - request: ToolParserRequest, - ) -> tuple[str, dict[str, Any]]: - raw_call = json.loads(body.strip()) - if not isinstance(raw_call, dict): - raise ValueError("IFM JSON tool call must be an object.") - - name = cls._validate_tool_name(raw_call.get("name"), request) - arguments = raw_call.get("arguments", {}) - if not isinstance(arguments, dict): - raise ValueError("IFM JSON tool-call arguments must be an object.") - - properties = find_tool_properties(request.tools, name) - arguments = { - arg_name: cls._coerce_json_value(arg_value, properties.get(arg_name)) - for arg_name, arg_value in arguments.items() - } - return name, arguments - - @classmethod - def _parse_xml_call( - cls, - body: str, - request: ToolParserRequest, - tool_format: str, - ) -> tuple[str, dict[str, Any]]: - first_arg = body.find(cls._ARG_KEY_START) - if first_arg == -1: - if any(marker in body for marker in cls._ARG_MARKERS): - raise ValueError("Malformed IFM XML argument tags.") - name = cls._validate_tool_name(body.strip(), request) - return name, {} - - name = cls._validate_tool_name(body[:first_arg].strip(), request) - properties = find_tool_properties(request.tools, name) - arguments: dict[str, Any] = {} - position = first_arg - for match in cls._ARG_RE.finditer(body, first_arg): - if body[position : match.start()].strip(): - raise ValueError("Malformed IFM XML argument tags.") - - arg_name = match.group(1).strip() - if not arg_name: - raise ValueError("IFM XML argument is missing a name.") - if arg_name in arguments: - raise ValueError(f"Duplicate IFM XML argument: {arg_name}.") - - explicit_type = match.group(2) - if explicit_type is not None: - explicit_type = explicit_type.strip() - if tool_format == "xml" and explicit_type is not None: - raise ValueError("Unexpected type tag in IFM XML tool call.") - if tool_format == "xml_typed" and not explicit_type: - raise ValueError("IFM typed XML argument is missing a type.") - arguments[arg_name] = cls._coerce_xml_value( - match.group(3), - properties.get(arg_name), - explicit_type, - ) - position = match.end() - - if position == first_arg or body[position:].strip(): - raise ValueError("Malformed IFM XML argument tags.") - return name, arguments - - @classmethod - def _parse_call( - cls, - body: str, - request: ToolParserRequest, - tool_format: str, - ) -> tuple[str, dict[str, Any]]: - if tool_format == "json": - return cls._parse_json_call(body, request) - return cls._parse_xml_call(body, request, tool_format) - - def extract_tool_calls( - self, - model_output: str, - request: ToolParserRequest, - ) -> ExtractedToolCallInformation: - tool_format = self._request_tool_format(request) - if request.tool_choice == "none": - return self._no_tool_calls(model_output) - - group_start = model_output.find(self._TOOL_CALLS_START) - if group_start == -1: - return self._no_tool_calls(model_output) - - body_start = group_start + len(self._TOOL_CALLS_START) - group_end = model_output.find(self._TOOL_CALLS_END, body_start) - if group_end == -1: - return self._no_tool_calls(model_output) - - group_body = model_output[body_start:group_end] - matches = list(self._TOOL_CALL_RE.finditer(group_body)) - if not matches or self._TOOL_CALL_RE.sub("", group_body).strip(): - logger.warning("Malformed K2 Horizon IFM tool-call group.") - return self._no_tool_calls(model_output) - - try: - tool_calls = [ - self._make_tool_call( - *self._parse_call(match.group(1), request, tool_format) - ) - for match in matches - ] - except (json.JSONDecodeError, TypeError, ValueError): - logger.warning("Failed to parse K2 Horizon IFM tool calls.", exc_info=True) - return self._no_tool_calls(model_output) - - suffix = model_output[group_end + len(self._TOOL_CALLS_END) :] - content = model_output[:group_start] + suffix - return ExtractedToolCallInformation( - tools_called=True, - tool_calls=tool_calls, - content=self._content_or_none(content), - ) - - def _emit_content(self, addition: str, *, final: bool = False) -> str | None: - if addition: - if self._content_seen: - return addition - self._pending_content += addition - if self._pending_content.strip(): - self._content_seen = True - content = self._pending_content - self._pending_content = "" - return content - if final: - self._pending_content = "" - return None - - def _streaming_tool_calls( - self, - tool_calls: list[ToolCall], - ) -> list[DeltaToolCall]: - deltas: list[DeltaToolCall] = [] - for index, tool_call in enumerate(tool_calls): - arguments_json = tool_call.function.arguments - arguments = json.loads(arguments_json) - self.prev_tool_call_arr.append( - {"name": tool_call.function.name, "arguments": arguments} - ) - self.streamed_args_for_tool.append(arguments_json) - deltas.append( - DeltaToolCall( - index=index, - id=tool_call.id, - type="function", - function=DeltaFunctionCall( - name=tool_call.function.name, - arguments=arguments_json, - ), - ) - ) - return deltas - - def extract_tool_calls_streaming( - self, - previous_text: str, - current_text: str, - delta_text: str, - previous_token_ids: Sequence[int], - current_token_ids: Sequence[int], - delta_token_ids: Sequence[int], - request: ToolParserRequest, - ) -> DeltaMessage | None: - del ( - previous_text, - current_text, - previous_token_ids, - current_token_ids, - delta_token_ids, - ) - - self._request_tool_format(request) - if request.tool_choice == "none": - emitted_content = self._emit_content(delta_text) - return ( - DeltaMessage(content=emitted_content) - if emitted_content is not None - else None - ) - - self._stream_text += delta_text - if self._stream_resolved: - emitted_content = self._emit_content(delta_text) - return ( - DeltaMessage(content=emitted_content) - if emitted_content is not None - else None - ) - - content: str | None = None - if self._stream_group_start is None: - group_start = self._stream_text.find( - self._TOOL_CALLS_START, self._stream_cursor - ) - if group_start == -1: - pending = self._stream_text[self._stream_cursor :] - overlap = partial_tag_overlap(pending, self._TOOL_CALLS_START) - sendable_end = len(self._stream_text) - overlap - content = self._emit_content( - self._stream_text[self._stream_cursor : sendable_end] - ) - self._stream_cursor = sendable_end - return DeltaMessage(content=content) if content is not None else None - - self._stream_group_start = group_start - content = self._emit_content( - self._stream_text[self._stream_cursor : group_start] - ) - self._stream_cursor = group_start - - group_start = self._stream_group_start - body_start = group_start + len(self._TOOL_CALLS_START) - group_end = self._stream_text.find(self._TOOL_CALLS_END, body_start) - if group_end == -1: - return DeltaMessage(content=content) if content is not None else None - - group_end += len(self._TOOL_CALLS_END) - group_text = self._stream_text[group_start:group_end] - parsed = self.extract_tool_calls(group_text, request) - self._stream_resolved = True - - if not parsed.tools_called: - failed_content = self._emit_content( - self._stream_text[self._stream_cursor :] - ) - self._stream_cursor = len(self._stream_text) - combined = (content or "") + (failed_content or "") - return DeltaMessage(content=combined) if combined else None - - tool_calls = self._streaming_tool_calls(parsed.tool_calls) - suffix = self._emit_content(self._stream_text[group_end:]) - self._stream_cursor = len(self._stream_text) - combined = (content or "") + (suffix or "") - return DeltaMessage(content=combined or None, tool_calls=tool_calls) - - def finish_streaming(self) -> DeltaMessage | None: - if not self._stream_resolved: - content = self._emit_content( - self._stream_text[self._stream_cursor :], final=True - ) - self._stream_cursor = len(self._stream_text) - self._stream_resolved = True - else: - content = self._emit_content("", final=True) - return DeltaMessage(content=content) if content is not None else None diff --git a/vllm/tool_parsers/kimi_k2_tool_parser.py b/vllm/tool_parsers/kimi_k2_tool_parser.py deleted file mode 100644 index d1cc483018..0000000000 --- a/vllm/tool_parsers/kimi_k2_tool_parser.py +++ /dev/null @@ -1,20 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from vllm.entrypoints.openai.chat_completion.protocol import ( - ChatCompletionRequest, -) -from vllm.entrypoints.openai.responses.protocol import ResponsesRequest -from vllm.parser.engine.registered_adapters import KimiK2ParserToolAdapter - - -class KimiK2ToolParser(KimiK2ParserToolAdapter): # type: ignore[valid-type, misc] - structural_tag_model = "kimi" - - def adjust_request( - self, - request: ChatCompletionRequest | ResponsesRequest, - ) -> ChatCompletionRequest | ResponsesRequest: - if request.tools and request.tool_choice != "none": - request.skip_special_tokens = False - return request diff --git a/vllm/tool_parsers/kimi_k3_tool_parser.py b/vllm/tool_parsers/kimi_k3_tool_parser.py deleted file mode 100644 index 15b5a22df8..0000000000 --- a/vllm/tool_parsers/kimi_k3_tool_parser.py +++ /dev/null @@ -1,399 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Tool-call parser for the Kimi K3 (XTML) chat format. - -This turns the generated XTML ``response`` and ``tools`` channels back into -OpenAI-compatible ``content`` and ``tool_calls``. - -K3 assistant tool calls live in a nested ``tools`` channel:: - - <|open|>tools<|sep|> - <|open|>call tool="python" index="1"<|sep|> - <|open|>argument key="code" type="string"<|sep|>print(1)<|close|>argument<|sep|> - <|open|>argument key="opts" type="object"<|sep|>{"a":1}<|close|>argument<|sep|> - <|close|>call<|sep|> - <|close|>tools<|sep|> - -where ``<|open|>``, ``<|close|>``, ``<|sep|>`` are dedicated special tokens. The plain -reply lives in a sibling ``response`` channel which is unwrapped into content. -``response`` always precedes ``tools`` (see the protocol channel grammar). - -Argument decoding mirrors the template's type tagging (inverse encoding): - * type="string" -> value is the RAW text, no unescaping (template emits it raw) - * other types -> value is JSON-decoded (number/boolean/null/object/array) - -Attribute values (``tool=``, ``index=``, ``key=``, ``type=``) ARE escaped on the -encode side (``&`` -> ``&``, ``"`` -> ``"``), so ``_attrs`` reverses -them on decode (``"`` before ``&`` -- reverse of encode order). - -Known limitation: because string argument and response bodies are emitted raw, -a value that literally contains ``<|close|>argument<|sep|>`` or -``<|close|>response<|sep|>`` is indistinguishable from a real closing marker. -""" - -import json -from collections.abc import Sequence - -import regex as re -from openai.types.responses import ToolChoiceCustom, ToolChoiceFunction - -from vllm.entrypoints.generate.base.protocol import ( - DeltaFunctionCall, - DeltaMessage, - DeltaToolCall, - ExtractedToolCallInformation, - FunctionCall, - ToolCall, -) -from vllm.entrypoints.openai.chat_completion.protocol import ( - ChatCompletionNamedToolChoiceParam, - ChatCompletionRequest, -) -from vllm.entrypoints.openai.responses.protocol import ResponsesRequest -from vllm.exceptions import VLLMValidationError -from vllm.logger import init_logger -from vllm.tokenizers import TokenizerLike -from vllm.tool_parsers.abstract_tool_parser import Tool, ToolParser - -logger = init_logger(__name__) - -_O, _C, _S = r"<\|open\|>", r"<\|close\|>", r"<\|sep\|>" -_TEXT_UNTIL_SEP = r"(?:(?!" + _S + r").)*?" - - -def _partial_tag_overlap(text: str, tag: str) -> int: - max_len = min(len(text), len(tag) - 1) - for n in range(max_len, 0, -1): - if text.endswith(tag[:n]): - return n - return 0 - - -class KimiK3ToolParser(ToolParser): - supports_required_and_named = False - # Enables the vLLM-side XTML structural tag builder - # (``get_kimi_k3_structural_tag`` in ``structural_tag_registry``). With - # ``VLLM_ENFORCE_STRICT_TOOL_CALLING`` on (default), ``_apply_structural_tag`` - # constrains generation to K3's ``<|open|>tools<|sep|>`` channel for - # ``required`` (and ``auto`` when a tool sets ``strict``) instead of the - # generic JSON guided decoding, which conflicts with the XTML format. - structural_tag_model = "kimi_k3" - - def __init__(self, tokenizer: TokenizerLike, tools: list[Tool] | None = None): - super().__init__(tokenizer, tools) - - self.tools_open = "<|open|>tools<|sep|>" - self.tools_close = "<|close|>tools<|sep|>" - self.response_open = "<|open|>response<|sep|>" - self.response_close = "<|close|>response<|sep|>" - - # Regexes operate on detokenized text. The XTML markers reach us as the - # literal strings <|open|>/<|close|>/<|sep|>. adjust_request keeps them - # from being stripped and suppresses spacing between adjacent token - # pieces. As DEFENSE-IN-DEPTH we still tolerate optional whitespace - # WITHIN a marker (e.g. "<|open|> tools <|sep|>") in case some serving - # path leaves vLLM's added-token spacing on -- this `\s*` is a no-op on - # clean input, so the normal byte-exact path is unaffected. We do NOT - # strip body whitespace, so clean content stays byte-exact. - # All bodies use non-greedy (.*?) so each block stops at its own first - # closing marker -- see the module-level KNOWN LIMITATION about - # literal-marker values. - self._tools_open_re = re.compile(_O + r"\s*tools\s*" + _S) - self._tools_close_re = re.compile(_C + r"\s*tools\s*" + _S) - self._response_open_re = re.compile(_O + r"\s*response\s*" + _S) - self._response_close_re = re.compile(_C + r"\s*response\s*" + _S) - self._message_close_re = re.compile(_C + r"\s*message\s*" + _S) - self._call_re = re.compile( - _O - + r"\s*call\s+(?P" - + _TEXT_UNTIL_SEP - + r")" - + _S - + r"(?P.*?)" - + _C - + r"\s*call\s*" - + _S, - re.DOTALL, - ) - self._arg_re = re.compile( - _O - + r"\s*argument\s+(?P" - + _TEXT_UNTIL_SEP - + r")" - + _S - + r"(?P.*?)" - + _C - + r"\s*argument\s*" - + _S, - re.DOTALL, - ) - # attr segment: key="value" (value already escaped on the encode side) - self._attr_re = re.compile(r'(?P\w+)="(?P[^"]*)"') - self._response_re = re.compile( - _O + r"\s*response\s*" + _S + r"(?P.*?)" + _C + r"\s*response\s*" + _S, - re.DOTALL, - ) - - # streaming state - self._sent_content_idx = 0 - self._sent_tool_call_count = 0 - - if not self.model_tokenizer: - raise ValueError( - "The model tokenizer must be passed to the ToolParser " - "constructor during construction." - ) - - def adjust_request( - self, request: ChatCompletionRequest | ResponsesRequest - ) -> ChatCompletionRequest | ResponsesRequest: - named = isinstance( - request.tool_choice, - ( - ChatCompletionNamedToolChoiceParam, - ToolChoiceFunction, - ToolChoiceCustom, - ), - ) - structured_outputs = getattr(request, "structured_outputs", None) - has_structural_tag = ( - structured_outputs is not None - and structured_outputs.structural_tag is not None - ) - if named and not has_structural_tag: - # Without the XTML structural tag there is no way to force the - # named call (the generic JSON guided-decoding path conflicts - # with the XTML channel format). - raise VLLMValidationError( - "Named tool choice for Kimi K3 requires strict tool calling " - "(VLLM_ENFORCE_STRICT_TOOL_CALLING) so the XTML structural " - "tag can force the call. Otherwise use `tool_choice` set to " - '"auto", "required", or "none".', - parameter="tool_choice", - value=request.tool_choice, - ) - - if request.tools and (request.tool_choice == "required" or named): - # K3 emits tool calls in XTML. When strict tool calling is enabled, - # DelegatingParser._apply_structural_tag has already attached the K3 - # XTML structural tag (structural_tag_model = "kimi_k3"). We return - # early to skip the generic parent path, which would otherwise attach - # JSON guided decoding when strict calling is off -- that JSON - # constraint conflicts with the <|open|>tools<|sep|> channel. - request.skip_special_tokens = False - if hasattr(request, "spaces_between_special_tokens"): - request.spaces_between_special_tokens = False - return request - - request = super().adjust_request(request) - # The XTML markers (<|open|>/<|close|>/<|sep|>, - # <|open|>response<|sep|> ...) must - # reach this parser as CONTIGUOUS literal text. Two request flags govern - # that, and vLLM's detokenizer couples them: - # effective_spaces_between = - # skip_special_tokens OR spaces_between_special_tokens - # (vllm/v1/engine/detokenizer.py). - # The detokenizer treats these control tokens as separate sub-texts and, - # when the effective flag is True, joins them with spaces -> - # "<|open|> response <|sep|>", which the regexes below would NOT match. - # Forcing BOTH flags off is the only way to suppress that spacing. We - # set both unconditionally: the response channel is unwrapped from these - # markers even with no tools. - request.skip_special_tokens = False - if hasattr(request, "spaces_between_special_tokens"): - request.spaces_between_special_tokens = False - return request - - def _attrs(self, s: str) -> dict[str, str]: - return { - m["k"]: m["v"].replace(""", '"').replace("&", "&") - for m in self._attr_re.finditer(s) - } - - def _decode_call(self, attrs: str, body: str) -> ToolCall | None: - """Decode one ``call`` block into a :class:`ToolCall`. - - ``attrs`` is the text between ``<|open|>call `` and ``<|sep|>`` (carries - ``tool=`` / ``index=``); ``body`` is the sequence of ``argument`` blocks. - Each argument is re-typed per its ``type=`` tag: strings pass through - raw, everything else is JSON-decoded (falling back to raw text if the - JSON is malformed, so a partial stream never raises). Returns ``None`` - when no tool name is present (an empty/garbage block is dropped). - """ - call_attrs = self._attrs(attrs) - tool_name = call_attrs.get("tool", "") - arguments: dict = {} - for arg_match in self._arg_re.finditer(body): - arg_attrs = self._attrs(arg_match["attrs"]) - key = arg_attrs.get("key", "") - arg_type = arg_attrs.get("type", "string") - raw_value = arg_match["val"] - if arg_type == "string": - arguments[key] = raw_value - else: - try: - arguments[key] = json.loads(raw_value) - except json.JSONDecodeError: - arguments[key] = raw_value - if not tool_name: - return None - return ToolCall( - type="function", - function=FunctionCall( - name=tool_name, - arguments=json.dumps(arguments, ensure_ascii=False), - ), - ) - - def _strip_response_content(self, text: str) -> str | None: - """Strip XTML response/message markers from generated response text. - - In chat serving, ``<|open|>response<|sep|>`` is often part of the prompt - generation prefix, so the model output may only contain the body plus - ``<|close|>response<|sep|>``. Handle both that consumed-prefix shape and a - complete ``<|open|>response<|sep|>... <|close|>response<|sep|>`` wrapper. - """ - m_open = self._response_open_re.search(text) - if m_open is not None: - m_close = self._response_close_re.search(text, m_open.end()) - if m_close is not None: - text = text[m_open.end() : m_close.start()] - else: - text = text[m_open.end() :] - else: - text = self._response_close_re.sub("", text) - text = self._message_close_re.sub("", text) - return text or None - - def _content(self, model_output: str, before: str) -> str | None: - # prefer the unwrapped response channel; else the text before the tools - m = self._response_re.search(model_output) - if m is not None: - return m["c"] or None - return self._strip_response_content(before) - - def _extract_response_content(self, current_text: str) -> str | None: - # Streaming response text is computed from the accumulated text. This is - # what keeps split markers from leaking: - # <|open|> / response / <|sep|>Hi -> emit only "Hi" after open closes - # Hi<|open|> / tools / <|sep|>... -> emit "Hi", hold the tools marker - # Hi<|close|> / response / <|sep|>... -> emit "Hi", hold the close marker - # Tool calls are even simpler: they are not emitted until a full - # <|close|>call<|sep|> is present in current_text. - m_open = self._response_open_re.search(current_text) - # In the normal chat path, the response-open marker may be consumed as - # the generation prefix. Then generated text starts directly with the - # response body or with <|close|>response<|sep|> before a tools channel. - body_start = m_open.end() if m_open is not None else 0 - m_tools = self._tools_open_re.search(current_text, body_start) - m_rclose = self._response_close_re.search(current_text, body_start) - tools_start = m_tools.start() if m_tools else -1 - response_end = m_rclose.start() if m_rclose else -1 - - candidates = [i for i in (tools_start, response_end) if i != -1] - if candidates: - sendable_idx = min(candidates) - else: - overlap = max( - _partial_tag_overlap(current_text, self.response_open), - _partial_tag_overlap(current_text, self.response_close), - _partial_tag_overlap(current_text, self.tools_open), - ) - sendable_idx = len(current_text) - overlap - - if sendable_idx <= body_start: - return None - if self._sent_content_idx < body_start: - self._sent_content_idx = body_start - if sendable_idx <= self._sent_content_idx: - return None - - content = current_text[self._sent_content_idx : sendable_idx] - self._sent_content_idx = sendable_idx - return content or None - - def extract_tool_calls( - self, model_output: str, request: ChatCompletionRequest - ) -> ExtractedToolCallInformation: - m_open = self._tools_open_re.search(model_output) - if m_open is None: - # no tools channel -> content is the response channel (unwrapped) - return ExtractedToolCallInformation( - tools_called=False, - tool_calls=[], - content=self._content(model_output, model_output), - ) - try: - before = model_output[: m_open.start()] - start = m_open.end() - m_close = self._tools_close_re.search(model_output, start) - section = ( - model_output[start:] - if m_close is None - else model_output[start : m_close.start()] - ) - - tool_calls = [ - tc - for m in self._call_re.finditer(section) - if (tc := self._decode_call(m["attrs"], m["body"])) is not None - ] - if not tool_calls: - return ExtractedToolCallInformation( - tools_called=False, - tool_calls=[], - content=self._content(model_output, before), - ) - return ExtractedToolCallInformation( - tools_called=True, - tool_calls=tool_calls, - content=self._content(model_output, before), - ) - except Exception: - logger.exception("Error extracting K3 tool calls.") - return ExtractedToolCallInformation( - tools_called=False, tool_calls=[], content=model_output - ) - - def extract_tool_calls_streaming( - self, - previous_text: str, - current_text: str, - delta_text: str, - previous_token_ids: Sequence[int], - current_token_ids: Sequence[int], - delta_token_ids: Sequence[int], - request: ChatCompletionRequest, - ) -> DeltaMessage | None: - # Conservative streaming: stream unwrapped response-channel text, then - # buffer tool calls and emit each call once its block closes. - content = self._extract_response_content(current_text) - - # tools channel is open: parse fully-closed calls we have not emitted yet - m_tools = self._tools_open_re.search(current_text) - if m_tools is None: - return DeltaMessage(content=content) if content else None - - section = current_text[m_tools.end() :] - calls = [ - tc - for m in self._call_re.finditer(section) - if (tc := self._decode_call(m["attrs"], m["body"])) is not None - ] - if len(calls) <= self._sent_tool_call_count: - return DeltaMessage(content=content) if content else None - new = calls[self._sent_tool_call_count :] - - deltas = [ - DeltaToolCall( - index=self._sent_tool_call_count + i, - id=tc.id, - type="function", - function=DeltaFunctionCall( - name=tc.function.name, arguments=tc.function.arguments - ).model_dump(exclude_none=True), - ) - for i, tc in enumerate(new) - ] - self._sent_tool_call_count = len(calls) - return DeltaMessage(content=content, tool_calls=deltas) diff --git a/vllm/tool_parsers/lfm2_tool_parser.py b/vllm/tool_parsers/lfm2_tool_parser.py deleted file mode 100644 index 2a0bab07e2..0000000000 --- a/vllm/tool_parsers/lfm2_tool_parser.py +++ /dev/null @@ -1,434 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -import ast -import json -from collections.abc import Sequence - -import regex as re - -import vllm.envs as envs -from vllm.entrypoints.generate.base.protocol import ( - DeltaMessage, - ExtractedToolCallInformation, -) -from vllm.entrypoints.openai.chat_completion.protocol import ( - ChatCompletionRequest, -) -from vllm.entrypoints.openai.responses.protocol import ResponsesRequest -from vllm.logger import init_logger -from vllm.tokenizers import TokenizerLike -from vllm.tool_parsers.abstract_tool_parser import ( - Tool, - ToolParser, -) -from vllm.tool_parsers.utils import ( - UnexpectedAstError, - compute_tool_delta, - contains_broken_string_literal, - escape_ctrl_chars_in_strings, - escape_nested_quotes_in_strings, - handle_single_tool, - make_valid_python, - normalize_leading_zero_ints, - rename_reserved_kwargs, - restore_reserved_kwarg_names, -) - -logger = init_logger(__name__) - -TOOL_CALL_START = "<|tool_call_start|>" -TOOL_CALL_END = "<|tool_call_end|>" - - -class Lfm2ToolParser(ToolParser): - """Tool call parser for LiquidAI LFM2/LFM2.5 models that produce pythonic - tool calls wrapped in <|tool_call_start|> and <|tool_call_end|> tokens. - - Example model output: - <|tool_call_start|>[get_weather(location="Paris")]<|tool_call_end|> - The weather in Paris is sunny. - - Used when --enable-auto-tool-choice --tool-call-parser lfm2 are all set. - """ - - TOOL_CALL_REGEX = re.compile(r"\[.*\]$", re.DOTALL) - - def __init__( - self, - tokenizer: TokenizerLike, - tools: list[Tool] | None = None, - ): - super().__init__(tokenizer, tools) - - self.tool_call_start_token_id = self.vocab.get(TOOL_CALL_START) - self.tool_call_end_token_id = self.vocab.get(TOOL_CALL_END) - - if self.tool_call_start_token_id is None or self.tool_call_end_token_id is None: - raise RuntimeError( - "LFM2 tool parser could not locate " - "<|tool_call_start|>/<|tool_call_end|> tokens in the " - "tokenizer!" - ) - - # Trailing content already emitted to the client. Used by the - # streaming path to suppress LFM2's frequent echo of the tool - # call body after the first <|tool_call_end|> while still - # allowing legitimate post-call prose through. - self._trailing_emitted: str = "" - - def adjust_request( - self, request: ChatCompletionRequest | ResponsesRequest - ) -> ChatCompletionRequest | ResponsesRequest: - request = super().adjust_request(request) - if request.tools and request.tool_choice != "none": - # The <|tool_call_start|>/<|tool_call_end|> sentinels are - # registered as special tokens in the LFM2/LFM2.5 tokenizer. - # With the default ``skip_special_tokens=True`` they are - # stripped from the decoded text before reaching this parser, - # so the tool block becomes invisible. Force the engine to - # preserve them when tool calling is enabled. - request.skip_special_tokens = False - return request - - # Rename for readability. This is NOT a tool id. - @property - def current_tool_index(self) -> int: - return self.current_tool_id - - @current_tool_index.setter - def current_tool_index(self, value: int) -> None: - self.current_tool_id = value - - @staticmethod - def _restore_reserved(tool_call): - """Restore parameter names renamed by ``rename_reserved_kwargs``.""" - arguments = json.loads(tool_call.function.arguments) - restored = restore_reserved_kwarg_names(arguments) - if restored != arguments: - tool_call.function.arguments = json.dumps(restored, ensure_ascii=False) - return tool_call - - @staticmethod - def _strip_echo(raw_after: str) -> str: - """Drop any orphan <|tool_call_end|> (and the preceding text) from - trailing content. LFM2 occasionally echoes the call body after the - first end token and caps it with a second end token; everything - through the last such orphan is model garbage, not user content.""" - last_orphan = raw_after.rfind(TOOL_CALL_END) - if last_orphan != -1: - return raw_after[last_orphan + len(TOOL_CALL_END) :] - return raw_after - - @classmethod - def _extract_tool_call_text( - cls, model_output: str - ) -> tuple[str | None, str | None]: - """Extract the pythonic call text and surrounding content. - - Returns (tool_text, content) where tool_text is the text between - the sentinel tokens and content is everything outside them. - """ - start_idx = model_output.find(TOOL_CALL_START) - if start_idx == -1: - return None, model_output - - end_idx = model_output.find(TOOL_CALL_END, start_idx) - if end_idx == -1: - # Incomplete — treat entire text after start as tool call - tool_text = model_output[start_idx + len(TOOL_CALL_START) :] - content_before = model_output[:start_idx].strip() - content = content_before or None - return tool_text, content - - tool_text = model_output[start_idx + len(TOOL_CALL_START) : end_idx] - content_before = model_output[:start_idx].strip() - content_after = cls._strip_echo( - model_output[end_idx + len(TOOL_CALL_END) :] - ).strip() - - content_parts = [] - if content_before: - content_parts.append(content_before) - if content_after: - content_parts.append(content_after) - content = "\n".join(content_parts) if content_parts else None - - return tool_text, content - - def extract_tool_calls( - self, model_output: str, request: ChatCompletionRequest - ) -> ExtractedToolCallInformation: - tool_text, content = self._extract_tool_call_text(model_output) - - if tool_text is None: - return ExtractedToolCallInformation( - tools_called=False, tool_calls=[], content=model_output - ) - - tool_text = tool_text.strip() - - is_tool_call_pattern = False - try: - is_tool_call_pattern = ( - self.TOOL_CALL_REGEX.match( - tool_text, - timeout=envs.VLLM_TOOL_PARSE_REGEX_TIMEOUT_SECONDS, - ) - is not None - ) - except TimeoutError: - logger.warning("Regex timeout occurred when matching tool call pattern.") - - if not is_tool_call_pattern: - return ExtractedToolCallInformation( - tools_called=False, tool_calls=[], content=model_output - ) - - try: - kw_renamed = False - try: - module = ast.parse(tool_text) - except (SyntaxError, ValueError): - # Progressive rewrites, each a no-op on already-valid text: - # escape raw control chars / NUL bytes inside string literals, - # strip leading zeros from int literals (month=07), close - # unambiguous nested quotes (command='sed -n '1,9p' f.py'), - # and rename reserved-keyword parameters (from=1; restored - # below). The first rewrite whose result parses wins. - escaped = escape_ctrl_chars_in_strings( - normalize_leading_zero_ints(tool_text) - ) - candidates = [escaped] - requoted, requote_changed = escape_nested_quotes_in_strings(escaped) - if requote_changed: - # Requoting can move raw control chars (newlines beyond - # the phantom close) inside the string; escape again. - candidates.append(escape_ctrl_chars_in_strings(requoted)) - renamed, kw_renamed = rename_reserved_kwargs(candidates[-1]) - if kw_renamed: - candidates.append(renamed) - for candidate in candidates: - try: - module = ast.parse(candidate) - break - except (SyntaxError, ValueError): - continue - else: - raise - parsed = getattr(module.body[0], "value", None) - # An empty block ([]) must not report tools_called=True with zero - # calls, so require at least one element. - if ( - isinstance(parsed, ast.List) - and parsed.elts - and all(isinstance(e, ast.Call) for e in parsed.elts) - ): - tool_calls = [ - handle_single_tool(e) # type: ignore - for e in parsed.elts - ] - if kw_renamed: - tool_calls = [self._restore_reserved(tc) for tc in tool_calls] - return ExtractedToolCallInformation( - tools_called=True, - tool_calls=tool_calls, - content=content, - ) - else: - raise UnexpectedAstError("Tool output must be a list of function calls") - except Exception: - logger.exception("Error in extracting tool call from response.") - return ExtractedToolCallInformation( - tools_called=False, tool_calls=[], content=model_output - ) - - def extract_tool_calls_streaming( - self, - previous_text: str, - current_text: str, - delta_text: str, - previous_token_ids: Sequence[int], - current_token_ids: Sequence[int], - delta_token_ids: Sequence[int], - request: ChatCompletionRequest, - ) -> DeltaMessage | None: - # If the tool call start token hasn't appeared yet, stream as content. - if TOOL_CALL_START not in current_text: - return DeltaMessage(content=delta_text) - - # Compute leading content (before <|tool_call_start|>) that arrived - # in this delta and hasn't been streamed yet. Without this, when the - # prefix and the start token land in the same delta the prefix is - # silently dropped — token-by-token streaming masked the bug because - # the prefix tokens always arrived in earlier deltas. - leading_content = "" - if TOOL_CALL_START not in previous_text: - start_idx = current_text.find(TOOL_CALL_START) - # previous_text contained no start token, so it has already been - # streamed via the no-start-token branch above. - leading_content = current_text[len(previous_text) : start_idx] - - has_end_in_current = TOOL_CALL_END in current_text - has_end_in_previous = TOOL_CALL_END in previous_text - - # Compute trailing content (after <|tool_call_end|>) not yet - # streamed. LFM2 frequently echoes the tool call body again - # after the first end token, capped with a second end token. - # Suppress that echo: - # - If a second <|tool_call_end|> has appeared, treat - # everything through the last one as garbage. - # - If the trailing starts with `[` or `<` (potential echo - # body or another sentinel) and no second end token has - # arrived yet, buffer it instead of emitting. - trailing_content = "" - if has_end_in_current: - end_idx = current_text.find(TOOL_CALL_END) + len(TOOL_CALL_END) - full_trailing = current_text[end_idx:] - stripped_trailing = self._strip_echo(full_trailing) - if stripped_trailing == full_trailing: - # No second end token yet — possibly mid-echo. - lstripped = full_trailing.lstrip() - if lstripped.startswith("[") or lstripped.startswith("<"): - # Suspect echo; hold off until resolved. - final_trailing = self._trailing_emitted - else: - final_trailing = full_trailing - else: - final_trailing = stripped_trailing - if final_trailing.startswith(self._trailing_emitted): - trailing_content = final_trailing[len(self._trailing_emitted) :] - self._trailing_emitted = final_trailing - - # If tools were already parsed in a prior delta, just stream any - # newly arrived trailing content. - if has_end_in_current and self.prev_tool_call_arr and has_end_in_previous: - if trailing_content: - return DeltaMessage(content=trailing_content) - return DeltaMessage(content="") - - # Extract the pythonic text between start and end tokens. - tool_text = current_text.split(TOOL_CALL_START, 1)[1] - # Strip the end token if present (entire call arrived at once). - if TOOL_CALL_END in tool_text: - tool_text = tool_text.split(TOOL_CALL_END, 1)[0] - # Leading whitespace after the start token would make every completion - # candidate an IndentationError (the non-streaming path strips it too). - tool_text = tool_text.lstrip() - - def _content_only_or_none() -> DeltaMessage | None: - """Return a content-only delta if any content arrived in this - chunk, otherwise None. Used on incremental-parse failure paths - so leading/trailing content is never silently dropped. - """ - combined = leading_content + trailing_content - return DeltaMessage(content=combined) if combined else None - - try: - # A raw control char inside a string argument would make every - # completion candidate a SyntaxError; escape them here rather than - # inside make_valid_python so the shared helper keeps its upstream - # behavior for the other pythonic parsers. Leading zeros in int - # literals (month=07) and parameters named after Python keywords - # (`from=1`) can never parse; rewrite those too. All rewrites are - # deterministic, so successive chunks stay consistent; keyword - # names are restored after decoding. - tool_text = escape_ctrl_chars_in_strings( - normalize_leading_zero_ints(tool_text) - ) - if has_end_in_current: - # Nested-quote recovery needs the final text (which quote - # closes a partial string is not stable across chunks) and - # must never touch text that already parses. - try: - ast.parse(tool_text) - except (SyntaxError, ValueError): - requoted_text, requote_changed = escape_nested_quotes_in_strings( - tool_text - ) - if requote_changed: - tool_text = escape_ctrl_chars_in_strings(requoted_text) - renamed_tool_text, kw_renamed = rename_reserved_kwargs(tool_text) - if kw_renamed: - tool_text = renamed_tool_text - - # A broken string (its first closing quote cannot close it — - # nested-quote text mid-arrival) makes every completion-based - # partial parse an implicit-concatenation misreading whose - # streamed prefix could never be retracted. Withhold deltas - # until the requote recovery above has produced sane text. - if contains_broken_string_literal(tool_text): - return _content_only_or_none() - - valid_and_added_text = make_valid_python(tool_text) - if valid_and_added_text is None: - return _content_only_or_none() - valid_text, added_text = valid_and_added_text - - module = ast.parse(valid_text) - parsed = getattr(module.body[0], "value", None) - if not isinstance(parsed, ast.List) or not all( - isinstance(e, ast.Call) for e in parsed.elts - ): - raise UnexpectedAstError("Tool output must be a list of function calls") - tool_calls = [ - handle_single_tool(e) # type: ignore - for e in parsed.elts - ] - if kw_renamed: - tool_calls = [self._restore_reserved(tc) for tc in tool_calls] - - tool_deltas = [] - for index, new_call in enumerate(tool_calls): - if index < self.current_tool_index: - continue - - self.current_tool_index = index - if len(self.streamed_args_for_tool) == index: - self.streamed_args_for_tool.append("") - - new_call_complete = ( - index < len(tool_calls) - 1 or ")]" not in added_text - ) - if new_call_complete: - self.current_tool_index += 1 - - withheld_suffix = added_text[:-2] if not new_call_complete else "" - if not new_call_complete and added_text[-2] == ")": - withheld_suffix = withheld_suffix + "}" - withheld_suffix = withheld_suffix.replace("'", '"') - delta = compute_tool_delta( - self.streamed_args_for_tool[index], - new_call, - index, - withheld_suffix, - ) - - if delta is not None: - tool_deltas.append(delta) - if ( - delta.function is not None - and delta.function.arguments is not None - ): - self.streamed_args_for_tool[index] += delta.function.arguments - - if tool_deltas and not self.prev_tool_call_arr: - self.prev_tool_call_arr = [{"arguments": {}}] - - combined_content = leading_content + trailing_content - - if tool_deltas or combined_content: - return DeltaMessage( - content=combined_content if combined_content else None, - tool_calls=tool_deltas, - ) - elif not added_text and self.current_tool_id > 0: - return DeltaMessage(content="") - else: - return None - except Exception: - logger.exception("Error trying to handle streaming tool call.") - logger.debug( - "Skipping chunk as a result of tool streaming extraction error" - ) - return _content_only_or_none() diff --git a/vllm/tool_parsers/ling3_tool_parser.py b/vllm/tool_parsers/ling3_tool_parser.py deleted file mode 100644 index 9b309b1eac..0000000000 --- a/vllm/tool_parsers/ling3_tool_parser.py +++ /dev/null @@ -1,11 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from __future__ import annotations - -from vllm.parser.ling3 import Ling3ParserToolAdapter - - -class Ling3ToolParser(Ling3ParserToolAdapter): # type: ignore[valid-type, misc] - supports_required_and_named = False - structural_tag_model = "glm_4_7" diff --git a/vllm/tool_parsers/llama4_pythonic_tool_parser.py b/vllm/tool_parsers/llama4_pythonic_tool_parser.py deleted file mode 100644 index fdffb48ff7..0000000000 --- a/vllm/tool_parsers/llama4_pythonic_tool_parser.py +++ /dev/null @@ -1,210 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -import ast -from collections.abc import Sequence - -import regex as re -from transformers import PreTrainedTokenizerBase - -import vllm.envs as envs -from vllm.entrypoints.generate.base.protocol import ( - DeltaMessage, - ExtractedToolCallInformation, -) -from vllm.entrypoints.openai.chat_completion.protocol import ( - ChatCompletionRequest, -) -from vllm.logger import init_logger -from vllm.tool_parsers.abstract_tool_parser import ( - Tool, - ToolParser, -) -from vllm.tool_parsers.utils import ( - UnexpectedAstError, - compute_tool_delta, - handle_single_tool, - make_valid_python, -) - -logger = init_logger(__name__) - - -class Llama4PythonicToolParser(ToolParser): - """Toolcall parser for Llama4 that produce tool calls in a pythonic style - Use --enable-auto-tool-choice --tool-call-parser llama4_pythonic - """ - - # TODO(mdepinet): Possible future improvements: - # 1. Support text + tools separated by either <|python_tag|> or \n\n - # 2. Support tools outside of a list (or separated by a semicolon). - # This depends on item 1 for consistent streaming. - # Neither of these are necessary for e.g. ToolACE, but both would help make - # Llama3.2 models more reliable. - - TOOL_CALL_REGEX = re.compile( - r"\[([a-zA-Z]+\w*\(([a-zA-Z]+\w*=.*,\s*)*([a-zA-Z]+\w*=.*\s)?\),\s*)*([a-zA-Z]+\w*\(([a-zA-Z]+\w*=.*,\s*)*([a-zA-Z]+\w*=.*\s*)?\)\s*)+\]", - re.DOTALL, - ) - - def __init__( - self, - tokenizer: PreTrainedTokenizerBase, - tools: list[Tool] | None = None, - ): - super().__init__(tokenizer, tools) - - # Rename for readability. This is NOT a tool id. - @property - def current_tool_index(self) -> int: - return self.current_tool_id - - @current_tool_index.setter - def current_tool_index(self, value: int) -> None: - self.current_tool_id = value - - def extract_tool_calls( - self, model_output: str, request: ChatCompletionRequest - ) -> ExtractedToolCallInformation: - """Extract the tool calls from a complete model response.""" - # remove <|python_start|> and <|python_end|> - # as Llama 4 model sometime will output those tokens - if model_output.startswith("<|python_start|>"): - model_output = model_output[len("<|python_start|>") :] - model_output = model_output.replace("<|python_end|>", "") - - is_tool_call_pattern = False - try: - is_tool_call_pattern = ( - self.TOOL_CALL_REGEX.match( - model_output, timeout=envs.VLLM_TOOL_PARSE_REGEX_TIMEOUT_SECONDS - ) - is not None - ) - except TimeoutError: - logger.warning("Regex timeout occurred when matching tool call pattern.") - logger.debug( - "Regex timeout occurred when matching user input: %s", model_output - ) - - if not is_tool_call_pattern: - return ExtractedToolCallInformation( - tools_called=False, tool_calls=[], content=model_output - ) - - try: - module = ast.parse(model_output) - parsed = getattr(module.body[0], "value", None) - if isinstance(parsed, ast.List) and all( - isinstance(e, ast.Call) for e in parsed.elts - ): - return ExtractedToolCallInformation( - tools_called=True, - tool_calls=[ - handle_single_tool(e) # type: ignore - for e in parsed.elts - ], - content=None, - ) - else: - raise UnexpectedAstError("Tool output must be a list of function calls") - except Exception: - logger.exception("Error in extracting tool call from response.") - # Treat as regular text - return ExtractedToolCallInformation( - tools_called=False, tool_calls=[], content=model_output - ) - - def extract_tool_calls_streaming( - self, - previous_text: str, - current_text: str, - delta_text: str, - previous_token_ids: Sequence[int], - current_token_ids: Sequence[int], - delta_token_ids: Sequence[int], - request: ChatCompletionRequest, - ) -> DeltaMessage | None: - if not current_text.startswith("[") and not current_text.startswith( - "<|python_start|>" - ): - return DeltaMessage(content=delta_text) - - try: - # remove <|python_start|> and <|python_end|> - if current_text.startswith("<|python_start|>"): - current_text = current_text[len("<|python_start|>") :] - if current_text.endswith("<|python_end|>"): - current_text = current_text[: current_text.rfind("<|python_end|>")] - valid_and_added_text = make_valid_python(current_text) - if valid_and_added_text is None: - return None - valid_text, added_text = valid_and_added_text - - module = ast.parse(valid_text) - parsed = getattr(module.body[0], "value", None) - if not isinstance(parsed, ast.List) or not all( - isinstance(e, ast.Call) for e in parsed.elts - ): - raise UnexpectedAstError("Tool output must be a list of function calls") - tool_calls = [ - handle_single_tool(e) # type: ignore - for e in parsed.elts - ] - - tool_deltas = [] - for index, new_call in enumerate(tool_calls): - if index < self.current_tool_index: - continue - - self.current_tool_index = index - if len(self.streamed_args_for_tool) == index: - self.streamed_args_for_tool.append("") - - new_call_complete = ( - index < len(tool_calls) - 1 or ")]" not in added_text - ) - if new_call_complete: - self.current_tool_index += 1 - - withheld_suffix = added_text[:-2] if not new_call_complete else "" - if not new_call_complete and added_text[-2] == ")": - # Function call is incomplete. Withhold the closing bracket. - withheld_suffix = withheld_suffix + "}" - # Strings get single quotes in the model-produced string. - # JSON requires double quotes. - withheld_suffix = withheld_suffix.replace("'", '"') - delta = compute_tool_delta( - self.streamed_args_for_tool[index], new_call, index, withheld_suffix - ) - - if delta is not None: - tool_deltas.append(delta) - if ( - delta.function is not None - and delta.function.arguments is not None - ): - self.streamed_args_for_tool[index] += delta.function.arguments - - # HACK: serving_chat.py inspects the internal state of tool parsers - # when determining its final streaming delta, automatically - # adding autocompleted JSON. - # These two lines avoid that nonsense while ensuring finish_reason - # is set to tool_calls when at least one tool is called. - if tool_deltas and not self.prev_tool_call_arr: - self.prev_tool_call_arr = [{"arguments": {}}] - - if tool_deltas: - return DeltaMessage(tool_calls=tool_deltas) - elif not added_text and self.current_tool_id > 0: - # Return an empty DeltaMessage once the tool calls are all done - # so that finish_reason gets set. - return DeltaMessage(content="") - else: - return None - except Exception: - logger.exception("Error trying to handle streaming tool call.") - logger.debug( - "Skipping chunk as a result of tool streaming extraction error" - ) - return None diff --git a/vllm/tool_parsers/llama_tool_parser.py b/vllm/tool_parsers/llama_tool_parser.py deleted file mode 100644 index 8696af4699..0000000000 --- a/vllm/tool_parsers/llama_tool_parser.py +++ /dev/null @@ -1,334 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -import json -from collections.abc import Sequence - -import partial_json_parser -import regex as re -from partial_json_parser.core.options import Allow -from transformers import PreTrainedTokenizerBase - -import vllm.envs as envs -from vllm.entrypoints.generate.base.protocol import ( - DeltaFunctionCall, - DeltaMessage, - DeltaToolCall, - ExtractedToolCallInformation, - FunctionCall, - ToolCall, -) -from vllm.entrypoints.openai.chat_completion.protocol import ( - ChatCompletionRequest, -) -from vllm.logger import init_logger -from vllm.renderers.chat_utils import make_tool_call_id -from vllm.tool_parsers.abstract_tool_parser import ( - Tool, - ToolParser, -) -from vllm.tool_parsers.utils import ( - find_common_prefix, - is_complete_json, - partial_json_loads, -) - -logger = init_logger(__name__) - - -class Llama3JsonToolParser(ToolParser): - """Tool call parser for Llama 3.x and 4 models intended for use with the - examples/tool_chat_template_llama.jinja template. - - Used when --enable-auto-tool-choice --tool-call-parser llama3_json or - llama4_json are set. - """ - - bot_token: str = "<|python_tag|>" - structural_tag_model = "llama" - # Simple regex to find opening braces - we'll use JSON decoder for parsing - # This handles arbitrary nesting depth correctly - tool_call_start_regex: re.Pattern = re.compile(r"\{") - json_decoder: json.JSONDecoder = json.JSONDecoder() - - def __init__( - self, - tokenizer: PreTrainedTokenizerBase, - tools: list[Tool] | None = None, - ): - super().__init__(tokenizer, tools) - - # initialize properties used for state when parsing tool calls in - # streaming mode - self.prev_tool_call_arr: list[dict] = [] - self.current_tool_id: int = -1 - self.current_tool_name_sent: bool = False - self.streamed_args_for_tool: list[ - str - ] = [] # map what has been streamed for each tool so far to a list - self.bot_token_id = self.vocab.get(self.bot_token) - if self.bot_token_id is None: - raise RuntimeError( - "Llama3JsonToolParser could not locate the bot token " - f"'{self.bot_token}' in the tokenizer." - ) - - def extract_tool_calls( - self, model_output: str, request: ChatCompletionRequest - ) -> ExtractedToolCallInformation: - """Extract the tool calls from a complete model response. - Only extracts JSON content and ignores any surrounding plain text. - Supports both single JSON and multiple JSONs separated by semicolons. - """ - # Quick check before running regex - if not (self.bot_token in model_output or "{" in model_output): - return ExtractedToolCallInformation( - tools_called=False, tool_calls=[], content=model_output - ) - - # Keep track of the end index of the last parsed JSON object - # so we don't parse inner brackets - end_index = -1 - tool_calls: list[ToolCall] = [] - - try: - for match in self.tool_call_start_regex.finditer( - model_output, timeout=envs.VLLM_TOOL_PARSE_REGEX_TIMEOUT_SECONDS - ): - start_index = match.start() - # Skip if this brace is inside a previously parsed JSON object - if start_index <= end_index: - continue - - try: - obj, json_end_index = self.json_decoder.raw_decode( - model_output[start_index:] - ) - end_index = start_index + json_end_index - - # raise KeyError if missing - name = obj["name"] - arguments_or_params = ( - obj["arguments"] if "arguments" in obj else obj["parameters"] - ) - - tool_calls.append( - ToolCall( - type="function", - function=FunctionCall( - name=name, - # function call args are JSON but as a string - arguments=json.dumps( - arguments_or_params, ensure_ascii=False - ), - ), - ) - ) - except KeyError as e: - # Missing required key - missing_key = str(e).strip("'\"") - logger.exception( - "Couldn't extract tool call from JSON response. " - "Required key '%s' not present. " - "Returning output in content with empty tool calls.", - missing_key, - ) - return ExtractedToolCallInformation( - tools_called=False, tool_calls=[], content=model_output - ) - except Exception: - # Any other error during parsing - logger.exception( - "Error in extracting tool call from response. " - "Returning output in content with empty tool calls" - ) - return ExtractedToolCallInformation( - tools_called=False, tool_calls=[], content=model_output - ) - except TimeoutError: - logger.warning("Regex timeout occurred when matching tool call pattern.") - logger.debug( - "Regex timeout occurred when matching user input: %s", model_output - ) - return ExtractedToolCallInformation( - tools_called=False, tool_calls=[], content=model_output - ) - - # If we have valid tool calls, return them normally - if tool_calls: - return ExtractedToolCallInformation( - tools_called=True, tool_calls=tool_calls, content=None - ) - - # No valid tool calls found - return ExtractedToolCallInformation( - tools_called=False, tool_calls=[], content=model_output - ) - - def extract_tool_calls_streaming( - self, - previous_text: str, - current_text: str, - delta_text: str, - previous_token_ids: Sequence[int], - current_token_ids: Sequence[int], - delta_token_ids: Sequence[int], - request: ChatCompletionRequest, - ) -> DeltaMessage | None: - if not ( - current_text.startswith(self.bot_token) or current_text.startswith("{") - ): - return DeltaMessage(content=delta_text) - - # bit mask flags for partial JSON parsing. If the name hasn't been - # sent yet, don't allow sending - # an incomplete string since OpenAI only ever (as far as I have - # seen) allows sending the entire tool/ function name at once. - flags = Allow.ALL if self.current_tool_name_sent else Allow.ALL & ~Allow.STR - try: - tool_call_arr = [] - is_complete = [] - try: - # depending on the prompt format the Llama model may or may not - # prefix the output with the <|python_tag|> token - start_idx = ( - len(self.bot_token) - if current_text.startswith(self.bot_token) - else 0 - ) - while start_idx < len(current_text): - (obj, end_idx) = partial_json_loads(current_text[start_idx:], flags) - is_complete.append( - is_complete_json(current_text[start_idx : start_idx + end_idx]) - ) - start_idx += end_idx + len("; ") - # depending on the prompt Llama can use - # either arguments or parameters - if "parameters" in obj: - assert "arguments" not in obj, ( - "model generated both parameters and arguments" - ) - obj["arguments"] = obj["parameters"] - tool_call_arr.append(obj) - except partial_json_parser.core.exceptions.MalformedJSON: - logger.debug("not enough tokens to parse into JSON yet") - return None - - # select as the current tool call the one we're on the state at - current_tool_call: dict = ( - tool_call_arr[self.current_tool_id] if len(tool_call_arr) > 0 else {} - ) - - # case -- if no tokens have been streamed for the tool, e.g. - # only the array brackets, stream nothing - if len(tool_call_arr) == 0: - return None - - # case: we are starting a new tool in the array - # -> array has > 0 length AND length has moved past cursor - elif ( - len(tool_call_arr) > 0 and len(tool_call_arr) > self.current_tool_id + 1 - ): - # if we're moving on to a new call, first make sure we - # haven't missed anything in the previous one that was - # auto-generated due to JSON completions, but wasn't - # streamed to the client yet. - if self.current_tool_id >= 0: - cur_arguments = current_tool_call.get("arguments") - if cur_arguments: - cur_args_json = json.dumps(cur_arguments, ensure_ascii=False) - sent = len(self.streamed_args_for_tool[self.current_tool_id]) - argument_diff = cur_args_json[sent:] - - logger.debug("got arguments diff: %s", argument_diff) - delta = DeltaMessage( - tool_calls=[ - DeltaToolCall( - index=self.current_tool_id, - function=DeltaFunctionCall( - arguments=argument_diff - ).model_dump(exclude_none=True), - ) - ] - ) - self.streamed_args_for_tool[self.current_tool_id] += ( - argument_diff - ) - else: - delta = None - else: - delta = None - # re-set stuff pertaining to progress in the current tool - self.current_tool_id = len(tool_call_arr) - 1 - self.current_tool_name_sent = False - self.streamed_args_for_tool.append("") - logger.debug("starting on new tool %d", self.current_tool_id) - return delta - - # if the current tool name hasn't been sent, send if available - # - otherwise send nothing - elif not self.current_tool_name_sent: - function_name = current_tool_call.get("name") - if function_name: - delta = DeltaMessage( - tool_calls=[ - DeltaToolCall( - index=self.current_tool_id, - type="function", - id=make_tool_call_id(), - function=DeltaFunctionCall( - name=function_name - ).model_dump(exclude_none=True), - ) - ] - ) - self.current_tool_name_sent = True - else: - delta = None - - # now we know we're on the same tool call and we're streaming - # arguments - else: - cur_arguments = current_tool_call.get("arguments") - delta = None - - if cur_arguments: - sent = len(self.streamed_args_for_tool[self.current_tool_id]) - cur_args_json = json.dumps(cur_arguments, ensure_ascii=False) - prev_arguments = self.prev_tool_call_arr[self.current_tool_id].get( - "arguments" - ) - - argument_diff = None - if is_complete[self.current_tool_id]: - argument_diff = cur_args_json[sent:] - elif prev_arguments: - prev_args_json = json.dumps(prev_arguments, ensure_ascii=False) - if cur_args_json != prev_args_json: - prefix = find_common_prefix(prev_args_json, cur_args_json) - argument_diff = prefix[sent:] - - if argument_diff is not None: - delta = DeltaMessage( - tool_calls=[ - DeltaToolCall( - index=self.current_tool_id, - function=DeltaFunctionCall( - arguments=argument_diff - ).model_dump(exclude_none=True), - ) - ] - ) - self.streamed_args_for_tool[self.current_tool_id] += ( - argument_diff - ) - - self.prev_tool_call_arr = tool_call_arr - return delta - - except Exception: - logger.exception("Error trying to handle streaming tool call.") - logger.debug( - "Skipping chunk as a result of tool streaming extraction error" - ) - return None diff --git a/vllm/tool_parsers/longcat_tool_parser.py b/vllm/tool_parsers/longcat_tool_parser.py deleted file mode 100644 index 919f801437..0000000000 --- a/vllm/tool_parsers/longcat_tool_parser.py +++ /dev/null @@ -1,24 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -import regex as re - -from vllm.tokenizers import TokenizerLike -from vllm.tool_parsers.abstract_tool_parser import Tool -from vllm.tool_parsers.hermes_tool_parser import Hermes2ProToolParser - - -class LongcatFlashToolParser(Hermes2ProToolParser): - structural_tag_model = "longcat" - - def __init__(self, tokenizer: TokenizerLike, tools: list[Tool] | None = None): - super().__init__(tokenizer, tools) - - self.tool_call_start_token: str = "" - self.tool_call_end_token: str = "" - - self.tool_call_regex = re.compile( - r"(.*?)" - r"|(.*)", - re.DOTALL, - ) diff --git a/vllm/tool_parsers/mimo_tool_parser.py b/vllm/tool_parsers/mimo_tool_parser.py deleted file mode 100644 index 3b44c5d989..0000000000 --- a/vllm/tool_parsers/mimo_tool_parser.py +++ /dev/null @@ -1,8 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from vllm.parser.engine.registered_adapters import MiMoParserToolAdapter - - -class MiMoToolParser(MiMoParserToolAdapter): # type: ignore[valid-type, misc] - structural_tag_model = "mimo" diff --git a/vllm/tool_parsers/minicpm5xml_tool_parser.py b/vllm/tool_parsers/minicpm5xml_tool_parser.py deleted file mode 100644 index 818b51c47f..0000000000 --- a/vllm/tool_parsers/minicpm5xml_tool_parser.py +++ /dev/null @@ -1,815 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -import json -from collections.abc import Sequence -from typing import Any - -import regex as re - -from vllm.entrypoints.generate.base.protocol import ( - DeltaFunctionCall, - DeltaMessage, - DeltaToolCall, - ExtractedToolCallInformation, - FunctionCall, - ToolCall, -) -from vllm.entrypoints.openai.chat_completion.protocol import ( - ChatCompletionRequest, - ChatCompletionToolsParam, -) -from vllm.entrypoints.openai.responses.protocol import ResponsesRequest -from vllm.logger import init_logger -from vllm.renderers.chat_utils import make_tool_call_id -from vllm.tokenizers import TokenizerLike -from vllm.tool_parsers.abstract_tool_parser import ( - Tool, - ToolParser, -) -from vllm.tool_parsers.utils import partial_tag_overlap, safe_literal_eval -from vllm.utils import random_uuid - -logger = init_logger(__name__) - -try: - from lxml import etree as ET # type: ignore - - _HAS_LXML = True -except Exception: # pragma: no cover - import xml.etree.ElementTree as ET # type: ignore - - _HAS_LXML = False - -_FUNC_NAME_V1_REGEX = re.compile(r"]*>") -_PARAM_WITH_NAME_REGEX = re.compile( - r"([\s\S]*?)", re.DOTALL -) -_PARAM_MISSING_NAME_REGEX = re.compile(r"]*\bname=)[^>]*>", re.DOTALL) -_FUNC_BLOCK_REGEX = re.compile(r"", re.DOTALL) - -# SentencePiece/GPT-style decoders may emit U+0120 (Ġ) / U+010A (Ċ). -_TOKENIZER_SPACE = "\u0120" -_TOKENIZER_NEWLINE = "\u010a" - - -def _normalize_model_output(text: str) -> str: - if ( - _TOKENIZER_SPACE not in text - and _TOKENIZER_NEWLINE not in text - and " or . - normalized = normalized.replace("" not in text: - return text - visible = text.rsplit("", 1)[-1].lstrip() - return visible if visible else text - - -def _streaming_args_snapshot(args_json: str, *, is_complete: bool) -> str: - """Return the streamed arguments prefix, omitting the closing brace until done.""" - if is_complete or not args_json.endswith("}"): - return args_json - return args_json[:-1] - - -def _streaming_args_diff( - prev_args: str, args_json: str, *, is_complete: bool -) -> str | None: - """Compute the next arguments fragment for OpenAI-style streaming accumulation.""" - if prev_args == "{}": - prev_args = "" - - target = _streaming_args_snapshot(args_json, is_complete=is_complete) - if not prev_args: - return target or None - if target == prev_args: - return None - if target.startswith(prev_args): - return target[len(prev_args) :] or None - - # Recover from a previously closed partial JSON snapshot. - if prev_args.endswith("}"): - prev_open = prev_args[:-1] - if target.startswith(prev_open): - return target[len(prev_open) :] or None - - return None - - -def _parse_arguments(json_value: str) -> tuple[Any, bool]: - try: - try: - parsed_value = json.loads(json_value) - except json.JSONDecodeError: - parsed_value = safe_literal_eval(json_value) - return parsed_value, True - except Exception: - return json_value, False - - -def _get_argument_type( - func_name: str, - arg_key: str, - name_to_tool: dict[str, ChatCompletionToolsParam], -) -> str | None: - tool = name_to_tool.get(func_name) - if tool is None or tool.function.parameters is None: - return None - props = tool.function.parameters.get("properties", {}) - if arg_key not in props: - return None - return props[arg_key].get("type") - - -def _coerce_argument_value( - func_name: str, - arg_key: str, - value: Any, - name_to_tool: dict[str, ChatCompletionToolsParam], -) -> Any: - arg_type = _get_argument_type(func_name, arg_key, name_to_tool) - if arg_type == "string": - return ( - value if isinstance(value, str) else json.dumps(value, ensure_ascii=False) - ) - if isinstance(value, str): - parsed_val, _ = _parse_arguments(value) - return parsed_val - return value - - -def _add_argument( - func_name: str, - key: str, - val_text: str, - arguments: dict[str, Any], - seen_keys: set[str], - allowed_props: set[str], - name_to_tool: dict[str, ChatCompletionToolsParam], -) -> bool: - """Add one argument; return False if it violates the tool schema.""" - # MiniCPM5 can emit OpenAI-style wrappers, e.g. - # {'id': '...'}. Keep known fields and - # ignore extra fields so valid tool calls do not fall back to long text. - if ( - key in {"properties", "arguments"} - and allowed_props - and key not in allowed_props - ): - parsed_val, parsed_ok = _parse_arguments(val_text) - if not parsed_ok or not isinstance(parsed_val, dict): - return False - added = False - for wrapped_key, wrapped_value in parsed_val.items(): - if wrapped_key not in allowed_props: - continue - if wrapped_key in seen_keys: - return False - seen_keys.add(wrapped_key) - arguments[wrapped_key] = _coerce_argument_value( - func_name, wrapped_key, wrapped_value, name_to_tool - ) - added = True - return added - - if allowed_props and key not in allowed_props: - return True - if key in seen_keys: - return False - seen_keys.add(key) - arguments[key] = _coerce_argument_value(func_name, key, val_text, name_to_tool) - return True - - -def _normalize_alias_tool_call( - func_name: str, - arguments: dict[str, Any], - tool_names: set[str], -) -> tuple[str, dict[str, Any]]: - """Map common MiniCPM5 alias tool names to the exposed tool schema.""" - if func_name in tool_names: - return func_name, arguments - - if func_name == "get_details_by_phone" and "get_customer_by_phone" in tool_names: - phone_number = arguments.get("phone_number") or arguments.get("phone") - if phone_number is not None: - return "get_customer_by_phone", {"phone_number": phone_number} - - if func_name == "get_details_by_name" and "get_customer_by_name" in tool_names: - full_name = arguments.get("full_name") or arguments.get("name") - dob = arguments.get("date_of_birth") or arguments.get("dob") - mapped = {} - if full_name is not None: - mapped["full_name"] = full_name - if dob is not None: - mapped["date_of_birth"] = dob - mapped["dob"] = dob - if mapped: - return "get_customer_by_name", mapped - - if ( - func_name - in { - "get_line_details", - "get_line_status", - "get_roaming_status", - } - and "get_details_by_id" in tool_names - ): - detail_id = arguments.get("line_id") or arguments.get("id") - if detail_id is not None: - return "get_details_by_id", {"id": detail_id} - - if func_name == "get_plan_details" and "get_details_by_id" in tool_names: - plan_id = arguments.get("plan_id") or arguments.get("id") - if plan_id is not None: - return "get_details_by_id", {"id": plan_id} - - if func_name == "enable_roaming" and "toggle_roaming" in tool_names: - line_id = arguments.get("line_id") or arguments.get("id") - if line_id is not None: - return "toggle_roaming", {"line_id": line_id, "enabled": True} - - if func_name == "disable_roaming" and "toggle_roaming" in tool_names: - line_id = arguments.get("line_id") or arguments.get("id") - if line_id is not None: - return "toggle_roaming", {"line_id": line_id, "enabled": False} - - if ( - func_name in {"add_refueled_data", "add_data_to_line"} - and "refuel_data" in tool_names - ): - line_id = arguments.get("line_id") or arguments.get("id") - amount = ( - arguments.get("amount_gb") - or arguments.get("gb") - or arguments.get("gb_amount") - or arguments.get("amount") - ) - mapped = {} - if "customer_id" in arguments: - mapped["customer_id"] = arguments["customer_id"] - if line_id is not None: - mapped["line_id"] = line_id - if amount is not None: - mapped["amount_gb"] = amount - if mapped: - return "refuel_data", mapped - - return func_name, arguments - - -def _build_tool_maps( - tools: list[ChatCompletionToolsParam] | None, -) -> tuple[ - set[str], - dict[str, set[str]], - dict[str, set[str]], - dict[str, ChatCompletionToolsParam], -]: - name_to_tool: dict[str, ChatCompletionToolsParam] = {} - name_to_allowed_props: dict[str, set[str]] = {} - name_to_required: dict[str, set[str]] = {} - - for tool in tools or []: - name = tool.function.name - if not name: - continue - name_to_tool[name] = tool - params = tool.function.parameters or {} - props = params.get("properties", {}) if isinstance(params, dict) else {} - name_to_allowed_props[name] = set(props.keys()) - required = params.get("required", []) if isinstance(params, dict) else [] - try: - name_to_required[name] = set(required) - except TypeError: - logger.warning( - "Failed to parse 'required' field for tool %s. " - "It should be a list of strings. Got: %s", - name, - required, - ) - name_to_required[name] = set() - - return ( - set(name_to_tool.keys()), - name_to_allowed_props, - name_to_required, - name_to_tool, - ) - - -def _parse_function_block( - block: str, - tool_names: set[str], - name_to_allowed_props: dict[str, set[str]], - name_to_required: dict[str, set[str]], - name_to_tool: dict[str, ChatCompletionToolsParam], -) -> dict[str, Any] | None: - """Return {name, parameters} if block is valid, else None.""" - func_name: str | None = None - arguments: dict[str, Any] = {} - parsed_ok = False - param_invalid = False - - try: - if _HAS_LXML: - try: - parser = ET.XMLParser(**{"strip_cdata": False}) # type: ignore[call-arg] - except TypeError: - parser = ET.XMLParser() - root = ET.fromstring(block, parser=parser) - else: - root = ET.fromstring(block) - - if root.tag == "function": - func_node = root - else: - func_node = root.find("function") if hasattr(root, "find") else None - - if func_node is not None: - func_name = (func_node.attrib.get("name") or "").strip() - - args_node = func_node.find("arguments") if func_node is not None else None - param_nodes: list[Any] = [] - if func_node is not None: - param_nodes = list(func_node.findall("param")) - if args_node is not None and not param_nodes: - param_nodes = list(args_node.findall("param")) - - if func_node is not None: - seen_keys: set[str] = set() - allowed_props = name_to_allowed_props.get(func_name or "", set()) - has_invalid_param = False - for param in param_nodes: - key = param.attrib.get("name") - if not key: - has_invalid_param = True - break - val_text = param.text or "" - if not _add_argument( - func_name or "", - key, - val_text, - arguments, - seen_keys, - allowed_props, - name_to_tool, - ): - has_invalid_param = True - break - if has_invalid_param: - arguments.clear() - param_invalid = True - parsed_ok = bool(func_name) - except Exception: - parsed_ok = False - - if not parsed_ok: - try: - m_fn = _FUNC_NAME_V1_REGEX.search(block) - if m_fn: - func_name = (m_fn.group(1) or "").strip() - has_invalid_param = bool(_PARAM_MISSING_NAME_REGEX.search(block)) - seen_keys = set() - allowed_props = name_to_allowed_props.get(func_name or "", set()) - for pm in _PARAM_WITH_NAME_REGEX.finditer(block): - key = pm.group(1).strip() - val_text = pm.group(2) or "" - if val_text.startswith(""): - val_text = val_text[len("")] - else: - val_text = val_text.strip() - if not _add_argument( - func_name or "", - key, - val_text, - arguments, - seen_keys, - allowed_props, - name_to_tool, - ): - has_invalid_param = True - break - if has_invalid_param: - arguments.clear() - param_invalid = True - parsed_ok = bool(func_name) - except Exception: - parsed_ok = False - - if not func_name or param_invalid: - return None - - func_name, arguments = _normalize_alias_tool_call(func_name, arguments, tool_names) - - if func_name not in tool_names: - return None - - req_props = name_to_required.get(func_name, set()) - if req_props and not req_props.issubset(arguments.keys()): - return None - - if not parsed_ok: - return None - - return {"name": func_name, "parameters": arguments} - - -def _parse_partial_params( - block: str, - func_name: str, - name_to_allowed_props: dict[str, set[str]], - name_to_tool: dict[str, ChatCompletionToolsParam], -) -> dict[str, Any]: - arguments: dict[str, Any] = {} - seen_keys: set[str] = set() - allowed_props = name_to_allowed_props.get(func_name, set()) - for pm in _PARAM_WITH_NAME_REGEX.finditer(block): - key = pm.group(1).strip() - if not key or key in seen_keys: - continue - val_text = pm.group(2) or "" - if val_text.startswith(""): - val_text = val_text[len("")] - else: - val_text = val_text.strip() - _add_argument( - func_name, - key, - val_text, - arguments, - seen_keys, - allowed_props, - name_to_tool, - ) - return arguments - - -class MiniCPM5XMLToolParser(ToolParser): - """MiniCPM5 XML tool parser.""" - - def __init__( - self, - tokenizer: TokenizerLike, - tools: list[Tool] | None = None, - ): - super().__init__(tokenizer, tools) - self.tool_call_start_token = " None: - self._processed_len = 0 - self.current_tool_id = -1 - self.current_tool_name_sent = False - self.prev_tool_call_arr = [] - self.streamed_args_for_tool = [] - - def adjust_request( - self, request: ChatCompletionRequest | ResponsesRequest - ) -> ChatCompletionRequest | ResponsesRequest: - request = super().adjust_request(request) - if request.tools and request.tool_choice != "none": - # Tool XML tags are special tokens in MiniCPM5; must not strip them - # before tool parsing (see internlm2/mistral vLLM tool parsers). - request.skip_special_tokens = False - return request - - def extract_tool_calls( - self, - model_output: str, - request: ChatCompletionRequest, - ) -> ExtractedToolCallInformation: - model_output = _normalize_model_output(model_output) - if self.tool_call_start_token not in model_output: - logger.debug("[MiniCPM5XMLToolParser] no last_end: - normal_parts.append(model_output[last_end : match.start()]) - - block = match.group(0) - parsed = _parse_function_block( - block, - tool_names, - name_to_allowed_props, - name_to_required, - name_to_tool, - ) - if parsed is not None: - tool_calls.append( - ToolCall( - id=f"call_{random_uuid()}", - type="function", - function=FunctionCall( - name=parsed["name"], - arguments=json.dumps( - parsed["parameters"], - ensure_ascii=False, - ), - ), - ) - ) - else: - normal_parts.append(block) - - last_end = match.end() - - if last_end < len(model_output): - normal_parts.append(model_output[last_end:]) - - content = _strip_thinking_content("".join(normal_parts).strip()) - - logger.debug( - "[MiniCPM5XMLToolParser] extracted %d tool calls", - len(tool_calls), - ) - - tools_called = len(tool_calls) > 0 - return ExtractedToolCallInformation( - tools_called=tools_called, - tool_calls=tool_calls, - content=None if tools_called else content, - ) - except Exception as e: - logger.error("Error in MiniCPM5XMLToolParser.extract_tool_calls: %s", e) - return ExtractedToolCallInformation( - tools_called=False, - tool_calls=[], - content=model_output, - ) - - def _emit_tool_args_delta( - self, - tool_index: int, - args_json: str, - *, - is_complete: bool = False, - ) -> DeltaMessage | None: - prev_args = ( - self.streamed_args_for_tool[tool_index] - if tool_index < len(self.streamed_args_for_tool) - else "" - ) - arg_diff = _streaming_args_diff( - prev_args, - args_json, - is_complete=is_complete, - ) - if not arg_diff: - return None - while len(self.streamed_args_for_tool) <= tool_index: - self.streamed_args_for_tool.append("") - self.streamed_args_for_tool[tool_index] = prev_args + arg_diff - return DeltaMessage( - tool_calls=[ - DeltaToolCall( - index=tool_index, - function=DeltaFunctionCall( - arguments=arg_diff, - ).model_dump(exclude_none=True), - ) - ], - ) - - def _start_tool_call(self, func_name: str) -> DeltaMessage: - self.current_tool_id += 1 - self.current_tool_name_sent = True - while len(self.streamed_args_for_tool) <= self.current_tool_id: - self.streamed_args_for_tool.append("") - while len(self.prev_tool_call_arr) <= self.current_tool_id: - self.prev_tool_call_arr.append({}) - self.prev_tool_call_arr[self.current_tool_id] = {"name": func_name} - return DeltaMessage( - tool_calls=[ - DeltaToolCall( - index=self.current_tool_id, - id=make_tool_call_id(), - type="function", - function=DeltaFunctionCall( - name=func_name, - ).model_dump(exclude_none=True), - ) - ], - ) - - def _process_complete_block_streaming( - self, - block: str, - request: ChatCompletionRequest, - ) -> DeltaMessage | None: - tool_names, name_to_allowed_props, name_to_required, name_to_tool = ( - _build_tool_maps(request.tools) - ) - parsed = _parse_function_block( - block, - tool_names, - name_to_allowed_props, - name_to_required, - name_to_tool, - ) - if parsed is None: - return DeltaMessage(content=block) - - args_json = json.dumps(parsed["parameters"], ensure_ascii=False) - func_name = parsed["name"] - - if not self.current_tool_name_sent: - self.current_tool_id += 1 - tool_index = self.current_tool_id - while len(self.streamed_args_for_tool) <= tool_index: - self.streamed_args_for_tool.append("") - while len(self.prev_tool_call_arr) <= tool_index: - self.prev_tool_call_arr.append({}) - self.streamed_args_for_tool[tool_index] = args_json - self.prev_tool_call_arr[tool_index] = { - "name": func_name, - "arguments": parsed["parameters"], - } - if not self.prev_tool_call_arr: - self.prev_tool_call_arr = [{"arguments": {}}] - return DeltaMessage( - tool_calls=[ - DeltaToolCall( - index=tool_index, - id=make_tool_call_id(), - type="function", - function=DeltaFunctionCall( - name=func_name, - arguments=args_json, - ).model_dump(exclude_none=True), - ) - ], - ) - - tool_index = self.current_tool_id - self.prev_tool_call_arr[tool_index]["arguments"] = parsed["parameters"] - delta = self._emit_tool_args_delta( - tool_index, - args_json, - is_complete=True, - ) - self.current_tool_name_sent = False - if delta: - if not self.prev_tool_call_arr: - self.prev_tool_call_arr = [{"arguments": {}}] - return delta - return DeltaMessage(content="") - - def _process_partial_block_streaming( - self, - block: str, - request: ChatCompletionRequest, - ) -> DeltaMessage | None: - tool_names, name_to_allowed_props, _, name_to_tool = _build_tool_maps( - request.tools - ) - if _PARAM_MISSING_NAME_REGEX.search(block): - return None - - match = _FUNC_NAME_V1_REGEX.search(block) - if not match: - return None - func_name = (match.group(1) or "").strip() - if func_name not in tool_names: - return None - - if not self.current_tool_name_sent: - return self._start_tool_call(func_name) - - arguments = _parse_partial_params( - block, - func_name, - name_to_allowed_props, - name_to_tool, - ) - if not arguments: - return None - args_json = json.dumps(arguments, ensure_ascii=False) - self.prev_tool_call_arr[self.current_tool_id]["arguments"] = arguments - delta = self._emit_tool_args_delta( - self.current_tool_id, - args_json, - is_complete=False, - ) - if delta and not self.prev_tool_call_arr: - self.prev_tool_call_arr = [{"arguments": {}}] - return delta - - def extract_tool_calls_streaming( - self, - previous_text: str, - current_text: str, - delta_text: str, - previous_token_ids: Sequence[int], - current_token_ids: Sequence[int], - delta_token_ids: Sequence[int], - request: ChatCompletionRequest, - ) -> DeltaMessage | None: - del previous_token_ids, current_token_ids, delta_token_ids - try: - current_text = _normalize_model_output(current_text) - if not previous_text: - self._reset_stream_state() - - if self.tool_call_start_token not in current_text: - if self._processed_len < len(current_text): - content = current_text[self._processed_len :] - self._processed_len = len(current_text) - return DeltaMessage(content=content) if content else None - return None - - for match in _FUNC_BLOCK_REGEX.finditer(current_text): - if match.end() <= self._processed_len: - continue - - if match.start() > self._processed_len: - gap = current_text[self._processed_len : match.start()] - self._processed_len = match.start() - return DeltaMessage(content=gap) if gap else None - - block = match.group(0) - delta = self._process_complete_block_streaming(block, request) - self._processed_len = match.end() - if delta is not None: - return delta - - remainder = current_text[self._processed_len :] - if not remainder: - if ( - not delta_text - and self.current_tool_id >= 0 - and not self.current_tool_name_sent - ): - return DeltaMessage(content="") - return None - - func_idx = remainder.find(self.tool_call_start_token) - if func_idx > 0: - gap = remainder[:func_idx] - self._processed_len += func_idx - return DeltaMessage(content=gap) - - if func_idx == -1: - overlap = partial_tag_overlap(remainder, self.tool_call_start_token) - if overlap: - return None - self._processed_len = len(current_text) - return DeltaMessage(content=remainder) if remainder else None - - partial_block = remainder[func_idx:] - if self.tool_call_end_token in partial_block: - end_idx = partial_block.rfind(self.tool_call_end_token) - complete_block = partial_block[ - : end_idx + len(self.tool_call_end_token) - ] - delta = self._process_complete_block_streaming(complete_block, request) - self._processed_len += func_idx + len(complete_block) - if delta is not None: - return delta - partial_block = partial_block[end_idx + len(self.tool_call_end_token) :] - if not partial_block.strip(): - return None - func_idx = partial_block.find(self.tool_call_start_token) - if func_idx == -1: - self._processed_len = len(current_text) - if partial_block: - return DeltaMessage(content=partial_block) - return None - partial_block = partial_block[func_idx:] - - return self._process_partial_block_streaming(partial_block, request) - except Exception: - logger.exception( - "Error in MiniCPM5XMLToolParser.extract_tool_calls_streaming" - ) - return None diff --git a/vllm/tool_parsers/minimax_m2_tool_parser.py b/vllm/tool_parsers/minimax_m2_tool_parser.py deleted file mode 100644 index 850732c555..0000000000 --- a/vllm/tool_parsers/minimax_m2_tool_parser.py +++ /dev/null @@ -1,8 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from vllm.parser.engine.registered_adapters import MinimaxM2ParserToolAdapter - - -class MinimaxM2ToolParser(MinimaxM2ParserToolAdapter): # type: ignore[valid-type, misc] - structural_tag_model = "minimax" diff --git a/vllm/tool_parsers/minimax_m3_tool_parser.py b/vllm/tool_parsers/minimax_m3_tool_parser.py deleted file mode 100644 index 6be0687273..0000000000 --- a/vllm/tool_parsers/minimax_m3_tool_parser.py +++ /dev/null @@ -1,10 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from vllm.parser.engine.registered_adapters import MinimaxM3ParserToolAdapter - - -class MinimaxM3ToolParser(MinimaxM3ParserToolAdapter): # type: ignore[valid-type, misc] - # Required and named tool choices parse the native M3 syntax instead of - # forcing JSON output. - supports_required_and_named = False diff --git a/vllm/tool_parsers/mistral_tool_parser.py b/vllm/tool_parsers/mistral_tool_parser.py deleted file mode 100644 index 01bddd4ed6..0000000000 --- a/vllm/tool_parsers/mistral_tool_parser.py +++ /dev/null @@ -1,49 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from __future__ import annotations - -from typing import TYPE_CHECKING - -from vllm.parser.engine.registered_adapters import MistralParserToolAdapter - -if TYPE_CHECKING: - from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest - from vllm.entrypoints.openai.responses.protocol import ResponsesRequest - - -class MistralToolParser(MistralParserToolAdapter): # type: ignore[valid-type, misc] - # Marked so is_mistral_tool_parser() in vllm.utils.mistral recognises - # this adapter and lets adjust_request() fire for tool_choice="none" - # on grammar-capable Mistral tokenizers. - IS_MISTRAL_TOOL_PARSER = True - - # Mistral emits tool calls in the [TOOL_CALLS]name[ARGS]{...} format, not the - # plain JSON array / bare-args that the serving layer's generic required and - # named handlers expect. Route required/named through this parser's own - # extraction (treated like "auto") so the [TOOL_CALLS] format is parsed - # correctly instead of crashing or dumping the raw envelope into arguments. - supports_required_and_named = False - - @property - def bot_token(self) -> str: - """Return the Mistral tool-call marker exposed by the legacy parser.""" - return self._parser_engine.bot_token - - @property - def bot_token_id(self) -> int | None: - """Return the token id for :attr:`bot_token`.""" - return self._parser_engine.bot_token_id - - def adjust_request( - self, - request: ChatCompletionRequest | ResponsesRequest, - ) -> ChatCompletionRequest | ResponsesRequest: - # Skip the base ToolParser tool_choice -> structured_outputs - # conversion. Mistral enforces tool_choice via the grammar ``mode`` - # (auto/none/required/named); forwarding a tool-derived json_schema - # would be unioned into the grammar by mistral-common, allowing raw - # JSON in place of a real [TOOL_CALLS] call. Genuine user-provided - # structured output on the request is still honored by the engine's - # adjust_request. - return self._parser_engine.adjust_request(request) diff --git a/vllm/tool_parsers/muse_glimmer_tool_parser.py b/vllm/tool_parsers/muse_glimmer_tool_parser.py deleted file mode 100644 index 644ca5ed3e..0000000000 --- a/vllm/tool_parsers/muse_glimmer_tool_parser.py +++ /dev/null @@ -1,506 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""ATEM tool-call parser for MuseGlimmer. - -Faithful port of the MuseGlimmer ``response_schema`` tool-call contract from the -HuggingFace MuseGlimmer export (``convert_muse_glimmer_weights_to_hf.py``: -``MUSE_GLIMMER_RESPONSE_SCHEMA``). - -MuseGlimmer emits tool calls in an XML-ish ATEM format inside channel-scoped messages: - - <|start|>assistant to=self<|message|>...reasoning...<|eom|> - <|start|>assistant to=.<|message|> - - - value - - <|eom|> # non-final call - <|start|>assistant to=user<|message|>...final answer...<|eot|> - -Channel scoping is essential: an ```` echoed inside a ``to=self`` -reasoning block or a ``to=user`` final answer must NOT be parsed as a real tool -call. - -Rather than *subtracting* reasoning/answer spans with regex substitutions (the -approach the HF ``response_schema`` uses, which is safe only on a complete, -well-formed turn), this parser *segments* the output into messages and then -*selects* the tool-channel bodies. On a complete turn the two are equivalent; -on a truncated or damaged turn, subtraction can delete a valid tool call -(an unterminated ``to=self`` block makes the non-greedy strip run to the next -``<|eom|>``, which belongs to the tool-call message) whereas selection cannot. - -Usage: ``--enable-auto-tool-choice --tool-call-parser muse_glimmer`` -""" - -from __future__ import annotations - -import json -from collections.abc import Iterator, Sequence - -import regex as re -from openai.types.responses import ToolChoiceCustom, ToolChoiceFunction -from transformers import PreTrainedTokenizerBase - -from vllm.entrypoints.generate.base.protocol import ( - DeltaFunctionCall, - DeltaMessage, - DeltaToolCall, - ExtractedToolCallInformation, - FunctionCall, - ToolCall, -) -from vllm.entrypoints.openai.chat_completion.protocol import ( - ChatCompletionNamedToolChoiceParam, - ChatCompletionRequest, -) -from vllm.entrypoints.openai.responses.protocol import ResponsesRequest -from vllm.logger import init_logger -from vllm.renderers.chat_utils import make_tool_call_id -from vllm.tool_parsers.abstract_tool_parser import ( - Tool, - ToolParser, -) - -logger = init_logger(__name__) - -# --- Message framing ------------------------------------------------------- -# An assistant message header. All three parts are optional except the -# <|message|> terminator: -# "<|start|>assistant to=get_weather<|message|>" -- after an <|eom|> boundary -# " to=self<|message|>" -- first message of a turn -# (the prompt already ended -# with "<|start|>assistant") -# "<|message|>" -- bare recipient (public CoT -# / untagged content) -_MSG_HEADER_RE = re.compile( - r"(?:<\|start\|>\s*assistant)?[^\S\n]*(?:to=(?P[A-Za-z0-9_.\-]+))?<\|message\|>" -) -_MSG_END_RE = re.compile(r"<\|eom\|>|<\|eot\|>") - -# Recipients whose bodies are NOT tool calls. -_REASONING_RECIPIENT = "self" -_USER_RECIPIENT = "user" - -# Structural markers that must never reach the client. A streamed body is held -# back by up to len(marker)-1 characters so a marker split across two chunks is -# not emitted as content. -_STRUCTURAL_MARKERS = ("<|eom|>", "<|eot|>", "<|start|>", "<|message|>") -_MAX_MARKER_LEN = max(len(m) for m in _STRUCTURAL_MARKERS) -# A trailing " to=NAME" that could still grow into a bare message header (the -# first message of a turn has no <|start|> prefix -- the prompt ends with -# "<|start|>assistant", so the model's first emitted text is " to=self<|message|>"). -_OPEN_TAIL_TO_RE = re.compile(r"[^\S\n]+to=[A-Za-z0-9_.\-]*$") - -# --- Tool-call extraction (unchanged from MUSE_GLIMMER_RESPONSE_SCHEMA) ------------- -_INVOKE_RE = re.compile(r"()", re.DOTALL) -_NAME_RE = re.compile(r']*?\bname="([^"]+)"') -_PARAM_RE = re.compile( - r']*?\bname="(?P[^"]+)"[^>]*?>(?P.*?)', - re.DOTALL, -) -_FUNCTION_CALLS_OPEN = "" - - -def _decode_value(raw: str): - """JSON-decode a parameter value when possible, else keep the raw string. - - Mirrors the schema's ``x-parser: json`` with ``allow_non_json: True``. - """ - try: - return json.loads(raw) - except (json.JSONDecodeError, ValueError): - return raw - - -def _iter_messages(text: str) -> Iterator[tuple[str | None, str, bool]]: - """Segment *text* into assistant messages. - - Yields ``(recipient, body, closed)`` per message, where ``recipient`` is - ``None`` for a bare ``<|message|>`` header and ``closed`` is False for a - message that has not (yet) seen ``<|eom|>`` / ``<|eot|>``. - - A message is also terminated by the start of the NEXT header. Without that, - a reasoning block whose ``<|eom|>`` is missing (truncation, or a chunk - dropped at the reasoning -> tool transition) would absorb the tool-call - message that follows it and the call would be lost -- the same defect the - subtractive regexes have. - """ - pos = 0 - while pos < len(text): - header = _MSG_HEADER_RE.search(text, pos) - if header is None: - return - body_start = header.end() - end = _MSG_END_RE.search(text, body_start) - nxt = _MSG_HEADER_RE.search(text, body_start) - body_end = end.start() if end is not None else len(text) - closed = end is not None - if nxt is not None and nxt.start() < body_end: - body_end = nxt.start() - closed = False - next_pos = nxt.start() - else: - next_pos = end.end() if end is not None else len(text) - body = text[body_start:body_end] - # A body can never legitimately contain <|start|>. Seeing one means the - # next header is only partially generated (its <|message|> has not - # arrived), so the regex above could not recognise it yet. Cut there, - # otherwise the streamed body would grow to include the next header and - # then shrink back once it completes. - start_tok = body.find("<|start|>") - if start_tok != -1: - body = body[:start_tok] - closed = False - yield header.group("rcpt"), body, closed - pos = next_pos - - -def _trailing_partial_marker_len(text: str) -> int: - """Length of the longest suffix of *text* that prefixes a structural marker.""" - max_overlap = min(len(text), _MAX_MARKER_LEN - 1) - for overlap in range(max_overlap, 0, -1): - suffix = text[-overlap:] - if any(marker.startswith(suffix) for marker in _STRUCTURAL_MARKERS): - return overlap - return 0 - - -def _safe_open_body(body: str) -> str: - """Trim the tail of a still-growing body to what is safe to emit now. - - Holds back anything that could still turn out to be structural, so the - emitted prefix only ever grows. Chunks under speculative decoding are large - enough that markers routinely straddle them. - """ - tail_to = _OPEN_TAIL_TO_RE.search(body) - if tail_to is not None: - return body[: tail_to.start()] - partial = _trailing_partial_marker_len(body) - return body[: len(body) - partial] if partial else body - - -class MuseGlimmerToolParser(ToolParser): - # MuseGlimmer emits ATEM markup around tool-call arguments. The generic - # named/required tool_choice path in vllm/parser/abstract_parser.py assigns - # the raw model text straight to FunctionCall.arguments, which leaks that - # framing and yields invalid JSON; the "required" branch then silently - # swallows the ValidationError and returns tool_calls=null. Opting out - # routes named/required through extract_tool_calls / - # extract_tool_calls_streaming -- the same path "auto" already uses. - supports_required_and_named = False - - def __init__( - self, - tokenizer: PreTrainedTokenizerBase, - tools: list[Tool] | None = None, - ) -> None: - super().__init__(tokenizer, tools) - # Streaming cursors. vLLM constructs one ToolParser per request, so - # instance state is per-stream. - self._streamed_content_len: int = 0 - self._streamed_reasoning_len: int = 0 - self._emitted_tool_calls: int = 0 - - def adjust_request( - self, request: ChatCompletionRequest | ResponsesRequest - ) -> ChatCompletionRequest | ResponsesRequest: - """Force special tokens through, and keep JSON guided decoding off. - - ``skip_special_tokens`` defaults to True on both ChatCompletionRequest - and ResponsesRequest. Every rule in this parser keys off ``<|message|>`` - / ``<|eom|>`` / ``<|eot|>`` / ``<|start|>``, so with the default the - channel framing is stripped before we see it: message segmentation - finds nothing, reasoning-channel invokes are indistinguishable from real - ones, and the raw ATEM markup falls through to the client as content. - Set it unconditionally and FIRST -- not only for tools requests, and not - relying on the reasoning parser to have set it. - - For required/named ``tool_choice`` the base hook installs a JSON schema - constraint (ToolParser.adjust_request -> get_json_schema_from_tools). - MuseGlimmer emits ATEM XML, so under that constraint it writes JSON *inside* - the tool channel -- ``[{"name": ...`` -- with no - ```` for this parser to find. Skip the base hook so those - choices decode natively, the same as "auto". - """ - request.skip_special_tokens = False - - tool_choice = getattr(request, "tool_choice", None) - if request.tools and ( - tool_choice == "required" - or isinstance( - tool_choice, - ( - ChatCompletionNamedToolChoiceParam, - ToolChoiceFunction, - ToolChoiceCustom, - ), - ) - ): - return request - return super().adjust_request(request) - - # ---------------- channel selection ---------------- - - @classmethod - def _tool_channel_text(cls, text: str) -> str: - """Concatenate the bodies of messages addressed to a tool. - - Falls back to the whole text when no message header is present at all -- - that means the framing never reached us (``skip_special_tokens`` was on, - or the chunk carrying the header was dropped upstream), and scanning - everything is strictly better than returning nothing. - """ - bodies = [ - body - for rcpt, body, _closed in _iter_messages(text) - if rcpt is not None - and rcpt != _REASONING_RECIPIENT - and rcpt != _USER_RECIPIENT - ] - if bodies: - return "\n".join(bodies) - if _MSG_HEADER_RE.search(text) is None and ( - _FUNCTION_CALLS_OPEN in text or " tuple[str, str, bool, bool]: - """Return ``(content, reasoning, content_open, reasoning_open)``. - - The ``*_open`` flags say whether that channel's LAST message is still - being generated; only then must the caller hold back a partial - structural marker. Tracking them per channel matters: a closed - reasoning block whose text happens to end in ``<`` would otherwise stay - permanently truncated while a later content message is open. - """ - content_parts: list[str] = [] - reasoning_parts: list[str] = [] - content_open = False - reasoning_open = False - for rcpt, body, closed in _iter_messages(text): - if rcpt == _REASONING_RECIPIENT: - reasoning_parts.append(body) - reasoning_open = not closed - elif rcpt is None or rcpt == _USER_RECIPIENT: - content_parts.append(body) - content_open = not closed - return ( - "".join(content_parts), - "".join(reasoning_parts), - content_open, - reasoning_open, - ) - - # ---------------- tool name binding ---------------- - - @staticmethod - def _registered_names(request: ChatCompletionRequest | None) -> set[str]: - """Names of the tools the client registered on this request.""" - names: set[str] = set() - tools = getattr(request, "tools", None) if request is not None else None - for t in tools or []: - fn = getattr(t, "function", None) or t - name = getattr(fn, "name", None) - if name is None and isinstance(fn, dict): - name = fn.get("name") - if name: - names.add(name) - return names - - @staticmethod - def _normalize_name(emitted: str, registered: set[str]) -> str: - """Map an emitted ATEM invoke name back to a registered tool name. - - When a client registers a BARE name (e.g. ``get_weather``) the shipped - chat template renders the valid recipient as ``"get_weather.*"``, and - the model duly emits ``get_weather.get_weather``. Collapsing that - doubled form is safe: head and tail are identical and the collapsed - name is registered. - - Anything else is passed through unchanged. Matching on the trailing - segment alone is NOT safe -- an emitted ``weather.get`` against a - registered ``{calendar.get}`` has a unique leaf match and would silently - dispatch the wrong tool. - """ - if not registered or emitted in registered: - return emitted - head, sep, tail = emitted.partition(".") - if sep and head == tail and head in registered: - return head - logger.warning( - "MuseGlimmer: emitted tool name %r does not match any registered tool; " - "passing through unchanged.", - emitted, - ) - return emitted - - @classmethod - def _parse_tool_calls( - cls, text: str, registered: set[str] | None = None - ) -> list[ToolCall]: - registered = registered or set() - scoped = cls._tool_channel_text(text) - tool_calls: list[ToolCall] = [] - for invoke in _INVOKE_RE.findall(scoped): - name_m = _NAME_RE.search(invoke) - if not name_m: - continue - name = cls._normalize_name(name_m.group(1), registered) - args: dict = {} - for pm in _PARAM_RE.finditer(invoke): - args[pm.group("key")] = _decode_value(pm.group("value")) - tool_calls.append( - ToolCall( - function=FunctionCall( - name=name, - arguments=json.dumps(args, ensure_ascii=False), - ) - ) - ) - return tool_calls - - @classmethod - def _extract_content(cls, text: str) -> str | None: - """Return the user-facing body, or the raw text when unframed.""" - content, reasoning, _c_open, _r_open = cls._visible_channels(text) - if content: - return content - # No framing at all -> the whole thing is plain content. - if not reasoning and _MSG_HEADER_RE.search(text) is None: - return text or None - return None - - # ---------------- non-streaming ---------------- - - def extract_tool_calls( - self, model_output: str, request: ChatCompletionRequest - ) -> ExtractedToolCallInformation: - if ( - _FUNCTION_CALLS_OPEN not in model_output - and "... parsed -- typically a truncated - # call (finish_reason='length'/abort). Log it: silently - # returning "no tool call" here is indistinguishable from the - # model choosing not to call one, which makes this failure mode - # invisible in production. - logger.warning( - "MuseGlimmer: tool channel opened but no complete " - "parsed (truncated tool call?); returning content only." - ) - return ExtractedToolCallInformation( - tools_called=False, - tool_calls=[], - content=self._extract_content(model_output), - ) - return ExtractedToolCallInformation( - tools_called=True, - tool_calls=tool_calls, - content=self._extract_content(model_output), - ) - except Exception: - logger.exception("Error extracting MuseGlimmer ATEM tool calls.") - return ExtractedToolCallInformation( - tools_called=False, tool_calls=[], content=model_output - ) - - # ---------------- streaming ---------------- - - def extract_tool_calls_streaming( - self, - previous_text: str, - current_text: str, - delta_text: str, - previous_token_ids: Sequence[int], - current_token_ids: Sequence[int], - delta_token_ids: Sequence[int], - request: ChatCompletionRequest, - ) -> DeltaMessage | None: - """Incremental ATEM streaming for tool calls AND content. - - This parser owns every delta once reasoning has ended: in - vllm/parser/abstract_parser.py::parse_delta the "pass through as - content" fallback is guarded by ``not self._in_tool_call_phase(state)``, - and ``_in_tool_call_phase`` is simply ``tool_parser is not None and - state.reasoning_ended``. So with a tool parser loaded that fallback is - dead code, and returning None here DISCARDS the delta. Anything we do - not emit -- including the ``to=user`` final answer -- never reaches the - client. Hence content is emitted here, not left to the reasoning parser. - - Tool calls are surfaced only when an ```` block becomes - complete: the XML is opaque until closed and MuseGlimmer parameters are not - incremental JSON, so there is nothing meaningful to stream before then. - """ - if not previous_text: - # First delta of the tool phase (parse_delta resets previous_text - # to "" when it hands the stream over). Reset the cursors. - self._streamed_content_len = 0 - self._streamed_reasoning_len = 0 - self._emitted_tool_calls = 0 - - try: - registered = self._registered_names(request) - calls = self._parse_tool_calls(current_text, registered) - content, reasoning, content_open, reasoning_open = self._visible_channels( - current_text - ) - - # Trim the tail of a channel that is still growing, so the emitted - # prefix never shrinks between deltas. - if content_open: - content = _safe_open_body(content) - if reasoning_open: - reasoning = _safe_open_body(reasoning) - - content_delta = content[self._streamed_content_len :] - reasoning_delta = reasoning[self._streamed_reasoning_len :] - - tool_deltas: list[DeltaToolCall] = [] - for i in range(self._emitted_tool_calls, len(calls)): - fn = calls[i].function - tool_deltas.append( - DeltaToolCall( - index=i, - type="function", - id=make_tool_call_id(), - function=DeltaFunctionCall( - name=fn.name, - arguments=fn.arguments, - ).model_dump(exclude_none=True), - ) - ) - - if not content_delta and not reasoning_delta and not tool_deltas: - return None - - self._streamed_content_len = len(content) - self._streamed_reasoning_len = len(reasoning) - self._emitted_tool_calls = len(calls) - - message = DeltaMessage() - if content_delta: - message.content = content_delta - if reasoning_delta: - message.reasoning = reasoning_delta - if tool_deltas: - message.tool_calls = tool_deltas - return message - except Exception: - logger.exception("Error extracting MuseGlimmer ATEM streaming tool calls.") - return None diff --git a/vllm/tool_parsers/olmo3_tool_parser.py b/vllm/tool_parsers/olmo3_tool_parser.py deleted file mode 100644 index cb7bc632ba..0000000000 --- a/vllm/tool_parsers/olmo3_tool_parser.py +++ /dev/null @@ -1,227 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -import ast -from collections.abc import Sequence - -import regex as re -from transformers import PreTrainedTokenizerBase - -import vllm.envs as envs -from vllm.entrypoints.generate.base.protocol import ( - DeltaMessage, - ExtractedToolCallInformation, -) -from vllm.entrypoints.openai.chat_completion.protocol import ( - ChatCompletionRequest, -) -from vllm.logger import init_logger -from vllm.tool_parsers.abstract_tool_parser import ( - Tool, - ToolParser, -) -from vllm.tool_parsers.utils import ( - UnexpectedAstError, - compute_tool_delta, - handle_single_tool, - make_valid_python, -) - -logger = init_logger(__name__) - - -class Olmo3PythonicToolParser(ToolParser): - """Tool call parser for Olmo 3 models that produce tool calls as - newline-separated pythonic strings. - Used when --enable-auto-tool-choice --tool-call-parser pythonic are all set - Code copied from pythonic_tool_parser.py and updated to handle - - newline separated pythonic tool calls. - - argument values being null/true/false instead of Pythonic literals. - """ - - # TODO(mdepinet): Possible future improvements: - # 1. Support text + tools separated by either <|python_tag|> or \n\n - # 2. Support tools outside of a list (or separated by a semicolon). - # This depends on item 1 for consistent streaming. - # Neither of these are necessary for e.g. ToolACE, but both would help make - # Llama3.2 models more reliable. - - TOOL_CALL_REGEX = re.compile( - r"\[([a-zA-Z]+\w*\(([a-zA-Z]+\w*=.*,\s*)*([a-zA-Z]+\w*=.*\s)?\),\s*)*([a-zA-Z]+\w*\(([a-zA-Z]+\w*=.*,\s*)*([a-zA-Z]+\w*=.*\s*)?\)\s*)+\]", - re.DOTALL, - ) - - def __init__( - self, - tokenizer: PreTrainedTokenizerBase, - tools: list[Tool] | None = None, - ): - super().__init__(tokenizer, tools) - - # Rename for readability. This is NOT a tool id. - @property - def current_tool_index(self) -> int: - return self.current_tool_id - - @current_tool_index.setter - def current_tool_index(self, value: int) -> None: - self.current_tool_id = value - - def extract_tool_calls( - self, model_output: str, request: ChatCompletionRequest - ) -> ExtractedToolCallInformation: - """Extract the tool calls from a complete model response.""" - original_model_output = model_output - # Remove xml tags. - match = re.search( - r"(.*?)", model_output, re.DOTALL - ) - if match: - model_output = match.group(1).strip() - # Make the newline separated function calls into a list. - model_output = ", ".join( - [line.strip() for line in model_output.splitlines() if line.strip()] - ) - model_output = f"[{model_output}]" - - is_tool_call_pattern = False - try: - is_tool_call_pattern = ( - self.TOOL_CALL_REGEX.match( - model_output, timeout=envs.VLLM_TOOL_PARSE_REGEX_TIMEOUT_SECONDS - ) - is not None - ) - except TimeoutError: - logger.warning("Regex timeout occurred when matching tool call pattern.") - logger.debug( - "Regex timeout occurred when matching user input: %s", model_output - ) - - if not is_tool_call_pattern: - return ExtractedToolCallInformation( - tools_called=False, tool_calls=[], content=original_model_output - ) - - try: - module = ast.parse(model_output) - parsed = getattr(module.body[0], "value", None) - if isinstance(parsed, ast.List) and all( - isinstance(e, ast.Call) for e in parsed.elts - ): - return ExtractedToolCallInformation( - tools_called=True, - tool_calls=[ - handle_single_tool(e) # type: ignore - for e in parsed.elts - ], - content=None, - ) - else: - raise UnexpectedAstError("Tool output must be a list of function calls") - except Exception: - logger.exception("Error in extracting tool call from response.") - # Treat as regular text - return ExtractedToolCallInformation( - tools_called=False, tool_calls=[], content=original_model_output - ) - - def extract_tool_calls_streaming( - self, - previous_text: str, - current_text: str, - delta_text: str, - previous_token_ids: Sequence[int], - current_token_ids: Sequence[int], - delta_token_ids: Sequence[int], - request: ChatCompletionRequest, - ) -> DeltaMessage | None: - # All function calls start with the tag. - # But since this is streaming, we may have seen only part of the tag. - if not current_text.startswith("<"): - return DeltaMessage(content=delta_text) - - try: - # Remove xml tags. - if current_text.startswith(""): - current_text = current_text[len("") :] - if current_text.endswith(""): - current_text = current_text[: -len("")] - - valid_and_added_text = make_valid_python(current_text) - if valid_and_added_text is None: - return None - valid_text, added_text = valid_and_added_text - - # Make the newline separated function calls into a list. - valid_text = ", ".join( - [line.strip() for line in valid_text.splitlines() if line.strip()] - ) - valid_text = f"[{valid_text}]" - module = ast.parse(valid_text) - parsed = getattr(module.body[0], "value", None) - if not isinstance(parsed, ast.List) or not all( - isinstance(e, ast.Call) for e in parsed.elts - ): - raise UnexpectedAstError( - "Tool output must be a sequence of newline-separated calls" - ) - tool_calls = [ - handle_single_tool(e) # type: ignore - for e in parsed.elts - ] - - tool_deltas = [] - for index, new_call in enumerate(tool_calls): - if index < self.current_tool_index: - continue - - self.current_tool_index = index - if len(self.streamed_args_for_tool) == index: - self.streamed_args_for_tool.append("") - - new_call_complete = index < len(tool_calls) - 1 or ")" not in added_text - if new_call_complete: - self.current_tool_index += 1 - - withheld_suffix = added_text[:-1] if not new_call_complete else "" - if not new_call_complete and added_text[-1] == ")": - # Function call is incomplete. Withhold the closing bracket. - withheld_suffix = withheld_suffix + "}" - # Strings get single quotes in the model-produced string. - # JSON requires double quotes. - withheld_suffix = withheld_suffix.replace("'", '"') - delta = compute_tool_delta( - self.streamed_args_for_tool[index], new_call, index, withheld_suffix - ) - - if delta is not None: - tool_deltas.append(delta) - if ( - delta.function is not None - and delta.function.arguments is not None - ): - self.streamed_args_for_tool[index] += delta.function.arguments - - # HACK: serving_chat.py inspects the internal state of tool parsers - # when determining its final streaming delta, automatically - # adding autocompleted JSON. - # These two lines avoid that nonsense while ensuring finish_reason - # is set to tool_calls when at least one tool is called. - if tool_deltas and not self.prev_tool_call_arr: - self.prev_tool_call_arr = [{"arguments": {}}] - - if tool_deltas: - return DeltaMessage(tool_calls=tool_deltas) - elif not added_text and self.current_tool_id > 0: - # Return an empty DeltaMessage once the tool calls are all done - # so that finish_reason gets set. - return DeltaMessage(content="") - else: - return None - except Exception: - logger.exception("Error trying to handle streaming tool call.") - logger.debug( - "Skipping chunk as a result of tool streaming extraction error" - ) - return None diff --git a/vllm/tool_parsers/phi4mini_tool_parser.py b/vllm/tool_parsers/phi4mini_tool_parser.py deleted file mode 100644 index 0ef25b35e0..0000000000 --- a/vllm/tool_parsers/phi4mini_tool_parser.py +++ /dev/null @@ -1,123 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -import json -from collections.abc import Sequence -from typing import Any - -import regex as re -from transformers import PreTrainedTokenizerBase - -from vllm.entrypoints.generate.base.protocol import ( - DeltaMessage, - ExtractedToolCallInformation, - FunctionCall, - ToolCall, -) -from vllm.entrypoints.openai.chat_completion.protocol import ( - ChatCompletionRequest, -) -from vllm.logger import init_logger -from vllm.renderers.chat_utils import make_tool_call_id -from vllm.tool_parsers.abstract_tool_parser import ( - Tool, - ToolParser, -) - -logger = init_logger(__name__) - - -class Phi4MiniJsonToolParser(ToolParser): - """Tool call parser for phi-4-mini models intended for use with the - examples/tool_chat_template_llama.jinja template. - - Used when --enable-auto-tool-choice --tool-call-parser phi4_mini_json - are all set - """ - - def __init__( - self, - tokenizer: PreTrainedTokenizerBase, - tools: list[Tool] | None = None, - ) -> None: - super().__init__(tokenizer, tools) - - # initialize properties used for state when parsing tool calls in - # streaming mode - self.prev_tool_call_arr: list[dict[str, Any]] = [] - self.current_tool_id: int = -1 - self.streamed_args_for_tool: list[ - str - ] = [] # map what has been streamed for each tool so far to a list - self.bot_token: str = "functools" - - def extract_tool_calls( - self, model_output: str, request: ChatCompletionRequest - ) -> ExtractedToolCallInformation: - """Extract the tool calls from a complete model response.""" - logger.debug("Model output: %s", model_output) - - pattern = r"functools\[(.*?)\]" - matches = re.search(pattern, model_output, re.DOTALL) - - if not matches: - logger.debug("No function calls found") - return ExtractedToolCallInformation( - tools_called=False, tool_calls=[], content=model_output - ) - - try: - function_call_arr: list[dict[str, Any]] = [] - try: - json_content = "[" + matches.group(1) + "]" - - function_call_arr = json.loads(json_content) - logger.debug( - "Successfully extracted %d function calls", len(function_call_arr) - ) - except json.JSONDecodeError as e: - logger.error( - "Failed to parse function calls from model output. Error: %s", - str(e), - ) - - tool_calls: list[ToolCall] = [ - ToolCall( - id=make_tool_call_id(), - type="function", - function=FunctionCall( - name=raw_function_call["name"], - # function call args are JSON but as a string - arguments=json.dumps( - raw_function_call["arguments"] - if "arguments" in raw_function_call - else raw_function_call["parameters"], - ensure_ascii=False, - ), - ), - ) - for raw_function_call in function_call_arr - ] - - # get any content before the tool call - ret = ExtractedToolCallInformation( - tools_called=True, tool_calls=tool_calls, content=None - ) - return ret - - except Exception: - return ExtractedToolCallInformation( - tools_called=False, tool_calls=[], content=model_output - ) - - def extract_tool_calls_streaming( - self, - previous_text: str, - current_text: str, - delta_text: str, - previous_token_ids: Sequence[int], - current_token_ids: Sequence[int], - delta_token_ids: Sequence[int], - request: ChatCompletionRequest, - ) -> DeltaMessage | None: - return None diff --git a/vllm/tool_parsers/plamo3_engine_tool_parser.py b/vllm/tool_parsers/plamo3_engine_tool_parser.py deleted file mode 100644 index f64999742e..0000000000 --- a/vllm/tool_parsers/plamo3_engine_tool_parser.py +++ /dev/null @@ -1,9 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from vllm.parser.engine.registered_adapters import Plamo3ParserToolAdapter - - -class Plamo3EngineToolParser(Plamo3ParserToolAdapter): # type: ignore[valid-type, misc] - structural_tag_model = "plamo3" - supports_required_and_named = False diff --git a/vllm/tool_parsers/poolside_v1_tool_parser.py b/vllm/tool_parsers/poolside_v1_tool_parser.py deleted file mode 100644 index 9989272647..0000000000 --- a/vllm/tool_parsers/poolside_v1_tool_parser.py +++ /dev/null @@ -1,610 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""GLM-4 Tool Call Parser with incremental string streaming support. - -This parser fixes the streaming issue reported in Issue #32829 where long string -parameters (e.g., file content with 4000+ characters of code) are buffered until -complete, causing multi-second delays before the user sees any content. - -The fix streams string values incrementally as they arrive, providing a true -streaming experience for long content. -""" - -import json -from collections.abc import Sequence -from typing import Any - -import partial_json_parser.core.complete -import regex as re -from openai.types.responses import ToolChoiceCustom, ToolChoiceFunction -from partial_json_parser.core.options import Allow - -from vllm.entrypoints.generate.base.protocol import ( - DeltaFunctionCall, - DeltaMessage, - DeltaToolCall, - ExtractedToolCallInformation, - FunctionCall, - ToolCall, -) -from vllm.entrypoints.openai.chat_completion.protocol import ( - ChatCompletionNamedToolChoiceParam, - ChatCompletionRequest, -) -from vllm.entrypoints.openai.responses.protocol import ( - ResponsesRequest, -) -from vllm.logger import init_logger -from vllm.renderers.chat_utils import make_tool_call_id -from vllm.tokenizers import TokenizerLike -from vllm.tool_parsers.abstract_tool_parser import ( - Tool, - ToolParser, -) -from vllm.tool_parsers.utils import safe_literal_eval - -logger = init_logger(__name__) - - -class PoolsideV1ToolParser(ToolParser): - """Tool parser for GLM-4 models with incremental string streaming. - - This parser emits tool-call deltas incrementally as arguments arrive. - For string-type parameters, content is streamed character-by-character - rather than waiting for the complete tag. - """ - - supports_required_and_named = False - - def __init__(self, tokenizer: TokenizerLike, tools: list[Tool] | None = None): - super().__init__(tokenizer, tools) - # Stateful streaming fields - self.current_tool_name_sent: bool = False - self.prev_tool_call_arr: list[dict[str, Any]] = [] - self.current_tool_id: int = -1 - self.streamed_args_for_tool: list[str] = [] - - self.tool_call_start_token: str = "" - self.tool_call_end_token: str = "" - self.arg_key_start: str = "" - self.arg_key_end: str = "" - self.arg_val_start: str = "" - self.arg_val_end: str = "" - - self.tool_calls_start_token = self.tool_call_start_token - - self.func_call_regex = re.compile(r".*?", re.DOTALL) - self.func_detail_regex = re.compile( - r"\s*([^\n<]+?)\s*\n?\s*(.*?)?", re.DOTALL - ) - self.func_arg_regex = re.compile( - r"(.*?)\s*(.*?)", re.DOTALL - ) - - if not self.model_tokenizer: - raise ValueError( - "The model tokenizer must be passed to the ToolParser " - "constructor during construction." - ) - - self.tool_call_start_token_id = self.vocab.get(self.tool_call_start_token) - self.tool_call_end_token_id = self.vocab.get(self.tool_call_end_token) - self._buffer: str = "" - - # Streaming state for incremental tool-call streaming - self._in_tool_call: bool = False - self._current_tool_name: str | None = None - self._pending_key: str | None = None - self._streaming_string_value: bool = False - self._tool_call_ids: list[str] = [] - self._args_started: list[bool] = [] - self._args_closed: list[bool] = [] - self._seen_keys: list[set[str]] = [] - - @staticmethod - def _deserialize(value: str) -> Any: - try: - return json.loads(value) - except json.JSONDecodeError: - pass - - try: - return safe_literal_eval(value) - except (ValueError, SyntaxError): - pass - - return value - - @staticmethod - def _json_escape_string_content(s: str) -> str: - """JSON-escape string content for incremental streaming. - - This escapes the content that goes INSIDE a JSON string (between quotes), - not including the surrounding quotes themselves. - """ - if not s: - return "" - return json.dumps(s, ensure_ascii=False)[1:-1] - - @staticmethod - def _is_string_type( - tool_name: str, - arg_name: str, - tools: list[Tool] | None, - ) -> bool: - if tools is None: - return False - for tool in tools: - # ChatCompletion tools nest under .function; Responses - # FunctionTool is flat (.name/.parameters at the top level). - fn = getattr(tool, "function", tool) - if getattr(fn, "name", None) != tool_name: - continue - params = getattr(fn, "parameters", None) - if params is None: - return False - arg_type = params.get("properties", {}).get(arg_name, {}).get("type", None) - return arg_type == "string" - logger.debug("No tool named '%s'.", tool_name) - return False - - @staticmethod - def _tools_enabled(request: ChatCompletionRequest) -> bool: - """Return whether tool parsing should be applied for this request.""" - try: - tools = getattr(request, "tools", None) - tool_choice = getattr(request, "tool_choice", None) - return bool(tools) and tool_choice != "none" - except Exception: - logger.exception("Failed to determine if tools are enabled.") - return False - - def adjust_request( - self, request: ChatCompletionRequest | ResponsesRequest - ) -> ChatCompletionRequest | ResponsesRequest: - """Adjust request parameters for tool call token handling. - - For required/named tool_choice, skip super().adjust_request() so it - does not install JSON guided decoding. These models emit XML tool - calls (per the chat template), which JSON guidance would break. - """ - if request.tools: - tc = request.tool_choice - if tc == "required" or isinstance( - tc, - ( - ChatCompletionNamedToolChoiceParam, - ToolChoiceFunction, - ToolChoiceCustom, - ), - ): - request.skip_special_tokens = False - return request - request = super().adjust_request(request) - if request.tools and request.tool_choice != "none": - # Ensure tool call tokens (, ) are not skipped - # during decoding. Even though they are not marked as special tokens, - # setting skip_special_tokens=False ensures proper handling in - # transformers 5.x where decoding behavior may have changed. - request.skip_special_tokens = False - return request - - def extract_tool_calls( - self, - model_output: str, - request: ChatCompletionRequest, - ) -> ExtractedToolCallInformation: - matched_tool_calls = self.func_call_regex.findall(model_output) - logger.debug("model_output: %s", model_output) - try: - tool_calls: list[ToolCall] = [] - for match in matched_tool_calls: - tc_detail = self.func_detail_regex.search(match) - if not tc_detail: - logger.warning( - "Failed to parse tool call details from: %s", - match, - ) - continue - tc_name = tc_detail.group(1).strip() - tc_args = tc_detail.group(2) - pairs = self.func_arg_regex.findall(tc_args) if tc_args else [] - arg_dct: dict[str, Any] = {} - for key, value in pairs: - arg_key = key.strip() - # Keep string values verbatim; whitespace is significant - # (e.g. code/file content). Only strip non-string types. - if self._is_string_type(tc_name, arg_key, request.tools): - arg_val = value - else: - arg_val = self._deserialize(value.strip()) - logger.debug("arg_key = %s, arg_val = %s", arg_key, arg_val) - arg_dct[arg_key] = arg_val - tool_calls.append( - ToolCall( - type="function", - function=FunctionCall( - name=tc_name, - arguments=json.dumps(arg_dct, ensure_ascii=False), - ), - ) - ) - except Exception: - logger.exception("Failed to extract tool call spec") - return ExtractedToolCallInformation( - tools_called=False, tool_calls=[], content=model_output - ) - else: - if len(tool_calls) > 0: - content: str | None = model_output[ - : model_output.find(self.tool_calls_start_token) - ] - # Normalize empty/whitespace-only content to None - if not content or not content.strip(): - content = None - return ExtractedToolCallInformation( - tools_called=True, tool_calls=tool_calls, content=content - ) - return ExtractedToolCallInformation( - tools_called=False, tool_calls=[], content=model_output - ) - - def extract_tool_calls_streaming( - self, - previous_text: str, - current_text: str, - delta_text: str, - previous_token_ids: Sequence[int], - current_token_ids: Sequence[int], - delta_token_ids: Sequence[int], - request: ChatCompletionRequest, - ) -> DeltaMessage | None: - if not self._tools_enabled(request): - return DeltaMessage(content=delta_text) if delta_text else None - - self._buffer += delta_text - - pending_deltas: dict[int, DeltaToolCall] = {} - content: str | None = None - - while True: - if not self._in_tool_call: - start_idx = self._buffer.find(self.tool_call_start_token) - if start_idx == -1: - # Check for partial start token at end of buffer - for i in range(1, len(self.tool_call_start_token)): - if self._buffer.endswith(self.tool_call_start_token[:i]): - out = self._buffer[:-i] - self._buffer = self._buffer[-i:] - if out: - content = (content or "") + out - break - else: - out = self._buffer - self._buffer = "" - if out: - content = (content or "") + out - break - - if start_idx > 0: - content = (content or "") + self._buffer[:start_idx] - self._buffer = self._buffer[start_idx:] - - self._buffer = self._buffer[len(self.tool_call_start_token) :] - self._begin_tool_call() - continue - - # Parse tool name first - if not self.current_tool_name_sent: - nl = self._buffer.find("\n") - ak = self._buffer.find(self.arg_key_start) - end = self._buffer.find(self.tool_call_end_token) - candidates = [i for i in [nl, ak, end] if i != -1] - if not candidates: - break - cut = min(candidates) - tool_name = self._buffer[:cut].strip() - if tool_name == "" and cut == end: - # Handle empty tool call like ``. - # Consume the tokens and reset state to avoid infinite loop. - self._buffer = self._buffer[end + len(self.tool_call_end_token) :] - self._finish_tool_call() - self._revert_last_tool_call_state() - continue - - if cut == nl: - self._buffer = self._buffer[nl + 1 :] - else: - self._buffer = self._buffer[cut:] - - self._current_tool_name = tool_name - self.current_tool_name_sent = True - self._update_tool_name(pending_deltas, tool_name) - continue - - assert self._current_tool_name is not None - - # Handle incremental string value streaming - if self._streaming_string_value: - val_end = self._buffer.find(self.arg_val_end) - if val_end != -1: - raw_content = self._buffer[:val_end] - self._buffer = self._buffer[val_end + len(self.arg_val_end) :] - self._streaming_string_value = False - self._pending_key = None - - escaped = self._json_escape_string_content(raw_content) - frag = escaped + '"' - self.streamed_args_for_tool[self.current_tool_id] += frag - self._update_tool_args(pending_deltas, frag) - continue - - # Check for partial at end - safe_len = len(self._buffer) - for i in range(1, len(self.arg_val_end)): - if self._buffer.endswith(self.arg_val_end[:i]): - safe_len = len(self._buffer) - i - break - - if safe_len > 0: - to_emit = self._buffer[:safe_len] - self._buffer = self._buffer[safe_len:] - escaped = self._json_escape_string_content(to_emit) - if escaped: - self.streamed_args_for_tool[self.current_tool_id] += escaped - self._update_tool_args(pending_deltas, escaped) - break - - # If we have a pending key, parse its value - if self._pending_key is not None: - val_pos = self._buffer.find(self.arg_val_start) - if val_pos == -1: - break - if val_pos > 0: - self._buffer = self._buffer[val_pos:] - - key = (self._pending_key or "").strip() - - is_string = self._is_string_type( - self._current_tool_name, key, request.tools - ) - - if is_string: - # String type: stream incrementally - self._buffer = self._buffer[len(self.arg_val_start) :] - - if key in self._seen_keys[self.current_tool_id]: - self._pending_key = None - continue - - self._seen_keys[self.current_tool_id].add(key) - key_json = json.dumps(key, ensure_ascii=False) - - if not self._args_started[self.current_tool_id]: - frag = "{" + key_json + ': "' - self._args_started[self.current_tool_id] = True - else: - frag = ", " + key_json + ': "' - - self.streamed_args_for_tool[self.current_tool_id] += frag - self._streaming_string_value = True - self._update_tool_args(pending_deltas, frag) - continue - - # Non-string type: wait for complete value - val_end = self._buffer.find(self.arg_val_end) - if val_end == -1: - break - - raw_val = self._buffer[len(self.arg_val_start) : val_end].strip() - self._buffer = self._buffer[val_end + len(self.arg_val_end) :] - self._pending_key = None - - frag_or_none = self._append_arg_fragment(key=key, raw_val=raw_val) - if frag_or_none: - self._update_tool_args(pending_deltas, frag_or_none) - continue - - # Parse next arg or close - end_pos = self._buffer.find(self.tool_call_end_token) - key_pos = self._buffer.find(self.arg_key_start) - if end_pos != -1 and (key_pos == -1 or end_pos < key_pos): - self._buffer = self._buffer[end_pos + len(self.tool_call_end_token) :] - frag_or_none = self._close_args_if_needed() - # Finalize prev_tool_call_arr with complete parsed arguments - if self._current_tool_name: - try: - full_args_str = self.streamed_args_for_tool[ - self.current_tool_id - ] - args_dict = json.loads(full_args_str) - self.prev_tool_call_arr[self.current_tool_id] = { - "name": self._current_tool_name, - "arguments": args_dict, - } - except (json.JSONDecodeError, IndexError) as e: - logger.warning( - "Failed to finalize tool call state for tool %d: %s", - self.current_tool_id, - e, - ) - self._finish_tool_call() - if frag_or_none: - self._update_tool_args(pending_deltas, frag_or_none) - continue - - if key_pos == -1: - break - if key_pos > 0: - self._buffer = self._buffer[key_pos:] - key_end = self._buffer.find(self.arg_key_end) - if key_end == -1: - break - key = self._buffer[len(self.arg_key_start) : key_end] - self._buffer = self._buffer[key_end + len(self.arg_key_end) :] - self._pending_key = key - continue - - tool_calls = list(pending_deltas.values()) - if content is None and len(tool_calls) == 0: - wants_logprobs = getattr(request, "logprobs", None) or ( - isinstance(request, ResponsesRequest) - and request.is_include_output_logprobs() - ) - if wants_logprobs: - return DeltaMessage(content="") - return None - return DeltaMessage(content=content, tool_calls=tool_calls) - - def _ensure_tool_state(self) -> None: - while len(self._tool_call_ids) <= self.current_tool_id: - self._tool_call_ids.append( - make_tool_call_id(id_type="random", func_name=None, idx=None) - ) - while len(self.streamed_args_for_tool) <= self.current_tool_id: - self.streamed_args_for_tool.append("") - while len(self.prev_tool_call_arr) <= self.current_tool_id: - self.prev_tool_call_arr.append({}) - while len(self._args_started) <= self.current_tool_id: - self._args_started.append(False) - while len(self._args_closed) <= self.current_tool_id: - self._args_closed.append(False) - while len(self._seen_keys) <= self.current_tool_id: - self._seen_keys.append(set()) - - def _begin_tool_call(self) -> None: - if self.current_tool_id == -1: - self.current_tool_id = 0 - else: - self.current_tool_id += 1 - self._ensure_tool_state() - self.current_tool_name_sent = False - self._current_tool_name = None - self._pending_key = None - self._streaming_string_value = False - self._in_tool_call = True - - def _finish_tool_call(self) -> None: - self._in_tool_call = False - self._current_tool_name = None - self._pending_key = None - self._streaming_string_value = False - - def _revert_last_tool_call_state(self) -> None: - """Revert the state allocation for the last tool call.""" - if self.current_tool_id < 0: - return - self._tool_call_ids.pop() - self.streamed_args_for_tool.pop() - self.prev_tool_call_arr.pop() - self._args_started.pop() - self._args_closed.pop() - self._seen_keys.pop() - self.current_tool_id -= 1 - - def _get_or_create_delta(self, pending: dict[int, DeltaToolCall]) -> DeltaToolCall: - idx = self.current_tool_id - if idx not in pending: - pending[idx] = DeltaToolCall( - index=idx, - function=DeltaFunctionCall(), - ) - delta = pending[idx] - assert delta.function is not None - return delta - - def _update_tool_name( - self, pending: dict[int, DeltaToolCall], tool_name: str - ) -> None: - self.prev_tool_call_arr[self.current_tool_id] = { - "name": self._current_tool_name, - "arguments": {}, - } - delta = self._get_or_create_delta(pending) - delta.id = self._tool_call_ids[self.current_tool_id] - delta.type = "function" - assert delta.function is not None - delta.function.name = tool_name - if delta.function.arguments is None: - delta.function.arguments = "" - - @staticmethod - def _complete_json_prefix( - json_prefix: str, - allowed_partial_types: Allow, - ) -> dict | None: - """Complete a partial JSON prefix into a valid JSON object. - - Returns (formatted_prefix, parsed_dict) or None on failure. - - Note: ``partial_json_parser`` strips trailing whitespace before - parsing (``complete.py:20``), which means the returned slice is - shorter than ``json_prefix`` when it has trailing whitespace. - Since the parser controls the construction of the json_prefix value, - this code relies on it being a valid prefix and we only use the fix for - the completion of the JSON object. - """ - try: - _, partial_str_completion = partial_json_parser.core.complete.fix( - json_prefix, - allowed_partial_types, - ) - return json.loads(json_prefix + partial_str_completion) - except Exception: - return None - - def _update_tool_args( - self, pending: dict[int, DeltaToolCall], fragment: str - ) -> None: - result = self._complete_json_prefix( - self.streamed_args_for_tool[self.current_tool_id], - Allow.ALL, - ) - if result is not None: - self.prev_tool_call_arr[self.current_tool_id]["arguments"] = result - delta = self._get_or_create_delta(pending) - assert delta.function is not None - if delta.function.arguments is None: - delta.function.arguments = "" - delta.function.arguments += fragment - - def _append_arg_fragment( - self, - *, - key: str, - raw_val: str, - ) -> str | None: - key = key.strip() - if not key: - return None - if key in self._seen_keys[self.current_tool_id]: - return None - - # This function is only called for non-string types (already checked - # by _is_string_type in the caller), so we always deserialize. - val_obj: Any = self._deserialize(raw_val) - - key_json = json.dumps(key, ensure_ascii=False) - val_json = json.dumps(val_obj, ensure_ascii=False) - - if not self._args_started[self.current_tool_id]: - fragment = "{" + key_json + ": " + val_json - self._args_started[self.current_tool_id] = True - else: - fragment = ", " + key_json + ": " + val_json - - self._seen_keys[self.current_tool_id].add(key) - self.streamed_args_for_tool[self.current_tool_id] += fragment - return fragment - - def _close_args_if_needed(self) -> str | None: - if self._args_closed[self.current_tool_id]: - return None - self._args_closed[self.current_tool_id] = True - if not self._args_started[self.current_tool_id]: - fragment = "{}" - self.streamed_args_for_tool[self.current_tool_id] = fragment - else: - fragment = "}" - self.streamed_args_for_tool[self.current_tool_id] += fragment - return fragment diff --git a/vllm/tool_parsers/pythonic_tool_parser.py b/vllm/tool_parsers/pythonic_tool_parser.py deleted file mode 100644 index 76b263cfc5..0000000000 --- a/vllm/tool_parsers/pythonic_tool_parser.py +++ /dev/null @@ -1,199 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -import ast -from collections.abc import Sequence - -import regex as re -from transformers import PreTrainedTokenizerBase - -import vllm.envs as envs -from vllm.entrypoints.generate.base.protocol import ( - DeltaMessage, - ExtractedToolCallInformation, -) -from vllm.entrypoints.openai.chat_completion.protocol import ( - ChatCompletionRequest, -) -from vllm.logger import init_logger -from vllm.tool_parsers.abstract_tool_parser import ( - Tool, - ToolParser, -) -from vllm.tool_parsers.utils import ( - UnexpectedAstError, - compute_tool_delta, - handle_single_tool, - make_valid_python, -) - -logger = init_logger(__name__) - - -class PythonicToolParser(ToolParser): - """Tool call parser for models that produce tool calls in a pythonic style, - such as Llama 3.2 and Llama 4 models. - - Used when --enable-auto-tool-choice --tool-call-parser pythonic are all set - """ - - # TODO(mdepinet): Possible future improvements: - # 1. Support text + tools separated by either <|python_tag|> or \n\n - # 2. Support tools outside of a list (or separated by a semicolon). - # This depends on item 1 for consistent streaming. - # Neither of these are necessary for e.g. ToolACE, but both would help make - # Llama3.2 models more reliable. - - TOOL_CALL_REGEX = re.compile( - r"\[([a-zA-Z]+\w*\(([a-zA-Z]+\w*=.*,\s*)*([a-zA-Z]+\w*=.*\s)?\),\s*)*([a-zA-Z]+\w*\(([a-zA-Z]+\w*=.*,\s*)*([a-zA-Z]+\w*=.*\s*)?\)\s*)+\]", - re.DOTALL, - ) - - def __init__( - self, - tokenizer: PreTrainedTokenizerBase, - tools: list[Tool] | None = None, - ): - super().__init__(tokenizer, tools) - - # Rename for readability. This is NOT a tool id. - @property - def current_tool_index(self) -> int: - return self.current_tool_id - - @current_tool_index.setter - def current_tool_index(self, value: int) -> None: - self.current_tool_id = value - - def extract_tool_calls( - self, model_output: str, request: ChatCompletionRequest - ) -> ExtractedToolCallInformation: - """Extract the tool calls from a complete model response.""" - is_tool_call_pattern = False - try: - is_tool_call_pattern = ( - self.TOOL_CALL_REGEX.match( - model_output, timeout=envs.VLLM_TOOL_PARSE_REGEX_TIMEOUT_SECONDS - ) - is not None - ) - except TimeoutError: - logger.warning("Regex timeout occurred when matching tool call pattern.") - logger.debug( - "Regex timeout occurred when matching user input: %s", model_output - ) - - if not is_tool_call_pattern: - return ExtractedToolCallInformation( - tools_called=False, tool_calls=[], content=model_output - ) - - try: - module = ast.parse(model_output) - parsed = getattr(module.body[0], "value", None) - if isinstance(parsed, ast.List) and all( - isinstance(e, ast.Call) for e in parsed.elts - ): - return ExtractedToolCallInformation( - tools_called=True, - tool_calls=[ - handle_single_tool(e) # type: ignore - for e in parsed.elts - ], - content=None, - ) - else: - raise UnexpectedAstError("Tool output must be a list of function calls") - except Exception: - logger.exception("Error in extracting tool call from response.") - # Treat as regular text - return ExtractedToolCallInformation( - tools_called=False, tool_calls=[], content=model_output - ) - - def extract_tool_calls_streaming( - self, - previous_text: str, - current_text: str, - delta_text: str, - previous_token_ids: Sequence[int], - current_token_ids: Sequence[int], - delta_token_ids: Sequence[int], - request: ChatCompletionRequest, - ) -> DeltaMessage | None: - if not current_text.startswith("["): - return DeltaMessage(content=delta_text) - - try: - valid_and_added_text = make_valid_python(current_text) - if valid_and_added_text is None: - return None - valid_text, added_text = valid_and_added_text - - module = ast.parse(valid_text) - parsed = getattr(module.body[0], "value", None) - if not isinstance(parsed, ast.List) or not all( - isinstance(e, ast.Call) for e in parsed.elts - ): - raise UnexpectedAstError("Tool output must be a list of function calls") - tool_calls = [ - handle_single_tool(e) # type: ignore - for e in parsed.elts - ] - - tool_deltas = [] - for index, new_call in enumerate(tool_calls): - if index < self.current_tool_index: - continue - - self.current_tool_index = index - if len(self.streamed_args_for_tool) == index: - self.streamed_args_for_tool.append("") - - new_call_complete = ( - index < len(tool_calls) - 1 or ")]" not in added_text - ) - if new_call_complete: - self.current_tool_index += 1 - - withheld_suffix = added_text[:-2] if not new_call_complete else "" - if not new_call_complete and added_text[-2] == ")": - # Function call is incomplete. Withhold the closing bracket. - withheld_suffix = withheld_suffix + "}" - # Strings get single quotes in the model-produced string. - # JSON requires double quotes. - withheld_suffix = withheld_suffix.replace("'", '"') - delta = compute_tool_delta( - self.streamed_args_for_tool[index], new_call, index, withheld_suffix - ) - - if delta is not None: - tool_deltas.append(delta) - if ( - delta.function is not None - and delta.function.arguments is not None - ): - self.streamed_args_for_tool[index] += delta.function.arguments - - # HACK: serving_chat.py inspects the internal state of tool parsers - # when determining its final streaming delta, automatically - # adding autocompleted JSON. - # These two lines avoid that nonsense while ensuring finish_reason - # is set to tool_calls when at least one tool is called. - if tool_deltas and not self.prev_tool_call_arr: - self.prev_tool_call_arr = [{"arguments": {}}] - - if tool_deltas: - return DeltaMessage(tool_calls=tool_deltas) - elif not added_text and self.current_tool_id > 0: - # Return an empty DeltaMessage once the tool calls are all done - # so that finish_reason gets set. - return DeltaMessage(content="") - else: - return None - except Exception: - logger.exception("Error trying to handle streaming tool call.") - logger.debug( - "Skipping chunk as a result of tool streaming extraction error" - ) - return None diff --git a/vllm/tool_parsers/response_template_tool_parser.py b/vllm/tool_parsers/response_template_tool_parser.py deleted file mode 100644 index 32b4b44de9..0000000000 --- a/vllm/tool_parsers/response_template_tool_parser.py +++ /dev/null @@ -1,6 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from vllm.parser.response_template import ResponseTemplateToolParser - -__all__ = ["ResponseTemplateToolParser"] diff --git a/vllm/tool_parsers/seed_oss_engine_tool_parser.py b/vllm/tool_parsers/seed_oss_engine_tool_parser.py deleted file mode 100644 index e708afd171..0000000000 --- a/vllm/tool_parsers/seed_oss_engine_tool_parser.py +++ /dev/null @@ -1,8 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from vllm.parser.engine.registered_adapters import SeedOssParserToolAdapter - - -class SeedOssEngineToolParser(SeedOssParserToolAdapter): # type: ignore[valid-type, misc] - structural_tag_model = None diff --git a/vllm/tool_parsers/step3_tool_parser.py b/vllm/tool_parsers/step3_tool_parser.py deleted file mode 100644 index 514945cd0d..0000000000 --- a/vllm/tool_parsers/step3_tool_parser.py +++ /dev/null @@ -1,306 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -import contextlib -import json -from collections.abc import Sequence -from typing import Any - -import regex as re - -from vllm.entrypoints.generate.base.protocol import ( - DeltaFunctionCall, - DeltaMessage, - DeltaToolCall, - ExtractedToolCallInformation, - FunctionCall, - ToolCall, -) -from vllm.entrypoints.openai.chat_completion.protocol import ( - ChatCompletionRequest, -) -from vllm.entrypoints.openai.responses.protocol import ResponsesRequest -from vllm.logger import init_logger -from vllm.tokenizers import TokenizerLike -from vllm.tool_parsers.abstract_tool_parser import ( - Tool, - ToolParser, -) -from vllm.utils import random_uuid - -logger = init_logger(__name__) - - -class Step3ToolParser(ToolParser): - """Tool parser for a model that uses a specific XML-like format for tool calls. - This version uses a robust, stateful, cursor-based streaming parser and - consolidates tool arguments into a single message. - """ - - TOOL_CALLS_BEGIN = "<|tool_calls_begin|>" - TOOL_CALLS_END = "<|tool_calls_end|>" - TOOL_CALL_BEGIN = "<|tool_call_begin|>" - TOOL_CALL_END = "<|tool_call_end|>" - TOOL_SEP = "<|tool_sep|>" - SPECIAL_TOKENS = [TOOL_CALLS_BEGIN, TOOL_CALLS_END, TOOL_CALL_BEGIN, TOOL_CALL_END] - - def __init__(self, tokenizer: TokenizerLike, tools: list[Tool] | None = None): - super().__init__(tokenizer, tools) - self.position = 0 - # Explicit state flags for robust streaming - self.tool_block_started = False - self.tool_block_finished = False - - def adjust_request( - self, request: ChatCompletionRequest | ResponsesRequest - ) -> ChatCompletionRequest | ResponsesRequest: - request = super().adjust_request(request) - if request.tools and request.tool_choice != "none": - request.skip_special_tokens = False - return request - - @staticmethod - def _parse_steptml_invoke( - action_text: str, - ) -> tuple[str | None, dict[str, str] | None]: - func_name_match = re.search(r'', action_text) - if not func_name_match: - return None, None - func_name = func_name_match.group(1) - - params: dict[str, str] = {} - param_matches = re.findall( - r'([^<]*)', - action_text, - ) - for name, value in param_matches: - params[name] = value.strip() - return func_name, params - - def _cast_arguments( - self, - func_name: str, - params: dict[str, Any], - ) -> dict[str, Any]: - for tool in self.tools or []: - if tool.function.name == func_name: - schema = tool.function.parameters or {} - properties = schema.get("properties", {}) - for key, value in params.items(): - if not isinstance(value, str): - continue - prop = properties.get(key, {}) - typ = prop.get("type") - if typ == "string": - params[key] = value.strip() - elif typ == "integer": - with contextlib.suppress(ValueError): - params[key] = int(value) - elif typ == "number": - with contextlib.suppress(ValueError): - params[key] = float(value) - elif typ == "boolean": - lower_val = value.lower() - params[key] = ( - lower_val == "true" - if lower_val in ("true", "false") - else value - ) - elif typ == "null": - params[key] = None if value.lower() == "null" else value - break - return params - - def extract_tool_calls_streaming( - self, - previous_text: str, - current_text: str, - delta_text: str, - previous_token_ids: Sequence[int], - current_token_ids: Sequence[int], - delta_token_ids: Sequence[int], - request: ChatCompletionRequest, - ) -> DeltaMessage | None: - # The main loop processes the stream from the last known position. - while True: - if self.position >= len(current_text): - return None # We've processed the entire stream. - - unprocessed_text = current_text[self.position :] - - # STATE: After all tools are done, all subsequent text is content. - if self.tool_block_finished: - self.position = len(current_text) - return DeltaMessage(content=unprocessed_text) - - # STATE: Before the tool block has started. - if not self.tool_block_started: - if unprocessed_text.startswith(self.TOOL_CALLS_BEGIN): - self.position += len(self.TOOL_CALLS_BEGIN) - self.tool_block_started = True - continue # Token consumed, re-loop. - - start_pos = unprocessed_text.find(self.TOOL_CALLS_BEGIN) - if start_pos == -1: - if ( - self.TOOL_CALLS_BEGIN.startswith(unprocessed_text.strip()) - and unprocessed_text - ): - return None # It's a prefix, wait. - self.position = len(current_text) - return DeltaMessage(content=unprocessed_text) - else: - content = unprocessed_text[:start_pos] - self.position += len(content) - return DeltaMessage(content=content) - - # STATE: Inside the main tool block. - offset = len(unprocessed_text) - len(unprocessed_text.lstrip()) - unprocessed_text = unprocessed_text.lstrip() - self.position += offset - - if unprocessed_text.startswith(self.TOOL_CALLS_END): - self.position += len(self.TOOL_CALLS_END) - self.tool_block_finished = True - self.current_tool_id = -1 - continue - - # Check if we are between tool calls. - tool_finished = self.current_tool_id != -1 and self.prev_tool_call_arr[ - self.current_tool_id - ].get("finished") - if self.current_tool_id == -1 or tool_finished: - if unprocessed_text.startswith(self.TOOL_CALL_BEGIN): - self.position += len(self.TOOL_CALL_BEGIN) - if self.current_tool_id == -1: - self.current_tool_id = 0 - else: - self.current_tool_id += 1 - self.current_tool_name_sent = False - while len(self.prev_tool_call_arr) <= self.current_tool_id: - self.prev_tool_call_arr.append({}) - self.prev_tool_call_arr[self.current_tool_id]["finished"] = False - continue - - if self.TOOL_CALL_BEGIN.startswith(unprocessed_text): - return None - - # STATE: Parsing an active tool call. - if self.current_tool_id != -1 and not self.prev_tool_call_arr[ - self.current_tool_id - ].get("finished", False): - end_tool_pos = unprocessed_text.find(self.TOOL_CALL_END) - if end_tool_pos == -1: - tool_body = unprocessed_text - else: - tool_body = unprocessed_text[:end_tool_pos] - - if end_tool_pos == -1 and self.TOOL_CALL_END.startswith(tool_body): - return None - - function_name, arguments = self._parse_steptml_invoke(tool_body) - if not function_name: - return None - - tool_call_arr = {"name": function_name, "parameters": arguments or {}} - - # Send the function name as soon as it's parsed. - if not self.current_tool_name_sent: - self.current_tool_name_sent = True - self.prev_tool_call_arr[self.current_tool_id].update(tool_call_arr) - return DeltaMessage( - tool_calls=[ - DeltaToolCall( - index=self.current_tool_id, - type="function", - id=f"chatcmpl-tool-{random_uuid()}", - function=DeltaFunctionCall(name=function_name), - ) - ] - ) - - # Update our internal state with the latest parsed arguments. - self.prev_tool_call_arr[self.current_tool_id].update( # noqa: E501 - tool_call_arr - ) - - # Only send arguments when the tool call is complete. - if end_tool_pos != -1: - self.position += end_tool_pos + len(self.TOOL_CALL_END) - self.prev_tool_call_arr[self.current_tool_id]["finished"] = True - - final_args = self._cast_arguments( - function_name, - tool_call_arr.get("parameters", {}), # type: ignore - ) - if final_args: - final_args_json = json.dumps(final_args, ensure_ascii=False) - return DeltaMessage( - tool_calls=[ - DeltaToolCall( - index=self.current_tool_id, - function=DeltaFunctionCall( - arguments=final_args_json - ), - ) - ] - ) - - # If tool is not finished, return None to wait for more tokens. - return None - - return None - - def extract_tool_calls( - self, - model_output: str, - request: ChatCompletionRequest, - ) -> ExtractedToolCallInformation: - if self.TOOL_CALLS_BEGIN not in model_output: - return ExtractedToolCallInformation( - tools_called=False, tool_calls=[], content=model_output - ) - - pre_text, rest = model_output.split(self.TOOL_CALLS_BEGIN, 1) - if self.TOOL_CALLS_END not in rest: - return ExtractedToolCallInformation( - tools_called=False, tool_calls=[], content=model_output - ) - - tool_block, post_text = rest.split(self.TOOL_CALLS_END, 1) - content = (pre_text + post_text).strip() - - tool_calls: list[ToolCall] = [] - call_parts = tool_block.split(self.TOOL_CALL_BEGIN) - - for part in call_parts: - if not part or self.TOOL_CALL_END not in part: - continue - - call_content = part.split(self.TOOL_CALL_END, 1)[0] - if self.TOOL_SEP not in call_content: - continue - - type_part, invoke_part = call_content.split(self.TOOL_SEP, 1) - if type_part.strip() != "function": - continue - - function_name, params_dict = self._parse_steptml_invoke(invoke_part) - - if function_name and params_dict is not None: - params_dict = self._cast_arguments(function_name, params_dict) - params_str = json.dumps(params_dict, ensure_ascii=False) - tool_calls.append( - ToolCall( - function=FunctionCall(name=function_name, arguments=params_str) - ) - ) - if tool_calls: - return ExtractedToolCallInformation( - tools_called=True, - tool_calls=tool_calls, - content=content if content else None, - ) - return ExtractedToolCallInformation( - tools_called=False, tool_calls=[], content=model_output - ) diff --git a/vllm/tool_parsers/step3p5_tool_parser.py b/vllm/tool_parsers/step3p5_tool_parser.py deleted file mode 100644 index 9d964ce314..0000000000 --- a/vllm/tool_parsers/step3p5_tool_parser.py +++ /dev/null @@ -1,8 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from vllm.parser.engine.registered_adapters import Step3p5ParserToolAdapter - - -class Step3p5ToolParser(Step3p5ParserToolAdapter): # type: ignore[valid-type, misc] - structural_tag_model = "qwen_3_coder" diff --git a/vllm/tool_parsers/xlam_tool_parser.py b/vllm/tool_parsers/xlam_tool_parser.py deleted file mode 100644 index dec34fc0b9..0000000000 --- a/vllm/tool_parsers/xlam_tool_parser.py +++ /dev/null @@ -1,561 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -# ruff: noqa -import json -from collections.abc import Sequence -from typing import Any, Optional, Union - -import regex as re - -from vllm.entrypoints.openai.chat_completion.protocol import ( - ChatCompletionRequest, -) -from vllm.renderers.chat_utils import make_tool_call_id -from vllm.entrypoints.generate.base.protocol import ( - DeltaFunctionCall, - DeltaMessage, - DeltaToolCall, - ExtractedToolCallInformation, - FunctionCall, - ToolCall, -) -from vllm.tool_parsers.abstract_tool_parser import ( - Tool, - ToolParser, -) -from vllm.logger import init_logger -from vllm.tokenizers import TokenizerLike -from vllm.utils import random_uuid - -logger = init_logger(__name__) - - -class xLAMToolParser(ToolParser): - def __init__(self, tokenizer: TokenizerLike, tools: list[Tool] | None = None): - super().__init__(tokenizer, tools) - - # Initialize state for streaming mode - self.prev_tool_calls: list[dict] = [] - self.current_tool_id = -1 - self.current_tool_name_sent = False - self.streamed_args: list[str] = [] # Track arguments sent for each tool - - # For backward compatibility with tests - self.current_tools_sent: list[bool] = [] - - # For backward compatibility with serving code - self.prev_tool_call_arr = [] - - # Regex patterns for preprocessing - self.json_code_block_patterns = [ - r"```(?:json)?\s*([\s\S]*?)```", - r"\[TOOL_CALLS\]([\s\S]*?)(?=\n|$)", - r"([\s\S]*?)", - ] - self.thinking_tag_pattern = r"([\s\S]*)" - - # Define streaming state type to be initialized later - self.streaming_state: dict[str, Any] = { - "current_tool_index": -1, - "tool_ids": [], - "sent_tools": [], - } - - def preprocess_model_output( - self, model_output: str - ) -> tuple[Optional[str], Optional[str]]: - """ - Preprocess the model output to extract content and potential tool calls. - Returns: - Tuple of (content, potential_tool_calls_json) - """ - # Check for thinking tag - thinking_match = re.search(self.thinking_tag_pattern, model_output) - if thinking_match: - content = model_output[: thinking_match.start() + len("")].strip() - thinking_content = thinking_match.group(1).strip() - - # Try to parse the thinking content as JSON - try: - json.loads(thinking_content) - return content, thinking_content - except json.JSONDecodeError: - # If can't parse as JSON, look for JSON code blocks - for json_pattern in self.json_code_block_patterns: - json_matches = re.findall(json_pattern, thinking_content) - if json_matches: - for json_str in json_matches: - try: - json.loads(json_str) - return content, json_str - except json.JSONDecodeError: - continue - - # Check for JSON code blocks in the entire output - for json_pattern in self.json_code_block_patterns: - json_matches = re.findall(json_pattern, model_output) - if json_matches: - for json_str in json_matches: - try: - json.loads(json_str) - # Extract content by removing the JSON code block - content = re.sub(json_pattern, "", model_output).strip() - return content, json_str - except json.JSONDecodeError: - continue - - # If the entire output is a valid JSON array or looks like one, treat it as tool calls - if model_output.strip().startswith("["): - try: - json.loads(model_output) - return None, model_output - except json.JSONDecodeError: - # Even if it's not valid JSON yet, it might be a tool call in progress - if ( - "{" in model_output - and "name" in model_output - and "arguments" in model_output - ): - return None, model_output - - # If no tool calls found, return the original output as content - return model_output, None - - def extract_tool_calls( - self, model_output: str, request: ChatCompletionRequest - ) -> ExtractedToolCallInformation: - """ - Extract tool calls from a complete model output. - """ - try: - # Preprocess the model output - content, potential_tool_calls = self.preprocess_model_output(model_output) - - if not potential_tool_calls: - return ExtractedToolCallInformation( - tools_called=False, tool_calls=[], content=content - ) - - # Parse the potential tool calls as JSON - tool_calls_data = json.loads(potential_tool_calls) - - # Ensure it's an array - if not isinstance(tool_calls_data, list): - logger.debug("Tool calls data is not an array") - return ExtractedToolCallInformation( - tools_called=False, - tool_calls=[], - content=content or model_output, - ) - - tool_calls: list[ToolCall] = [] - - for idx, call in enumerate(tool_calls_data): - if ( - not isinstance(call, dict) - or "name" not in call - or "arguments" not in call - ): - logger.debug("Invalid tool call format at index %d", idx) - continue - - tool_call = ToolCall( - id=f"call_{idx}_{random_uuid()}", - type="function", - function=FunctionCall( - name=call["name"], - arguments=( - json.dumps(call["arguments"], ensure_ascii=False) - if isinstance(call["arguments"], dict) - else call["arguments"] - ), - ), - ) - tool_calls.append(tool_call) - - return ExtractedToolCallInformation( - tools_called=len(tool_calls) > 0, - tool_calls=tool_calls, - content=content, - ) - - except Exception as e: - logger.exception("Error extracting tool calls: %s", str(e)) - return ExtractedToolCallInformation( - tools_called=False, tool_calls=[], content=model_output - ) - - def extract_tool_calls_streaming( - self, - previous_text: str, - current_text: str, - delta_text: str, - previous_token_ids: Sequence[int], - current_token_ids: Sequence[int], - delta_token_ids: Sequence[int], - request: ChatCompletionRequest, - ) -> Union[DeltaMessage, None]: - """ - Extract tool calls for streaming mode. - """ - # First, check for a definitive start of a tool call block. - # This prevents premature parsing of incomplete output. - stripped_text = current_text.strip() - preprocessed_content, preprocessed_tool_calls = self.preprocess_model_output( - current_text - ) - - # For JSON code blocks, we need to detect them earlier, even if incomplete - has_potential_json_block = ( - "```json" in current_text - or "```\n[" in current_text - or "[TOOL_CALLS]" in current_text - or "" in current_text - ) - - is_tool_call_block = ( - stripped_text.startswith("[") - or stripped_text.startswith("") - or stripped_text.startswith("[TOOL_CALLS]") - or - # Check if we have thinking tags with JSON-like content following - ("[" in current_text) - or - # Check if the text contains a JSON array after preprocessing - preprocessed_tool_calls is not None - or - # For JSON code blocks, detect early if we see enough structure - ( - has_potential_json_block - and '"name"' in current_text - and '"arguments"' in current_text - ) - ) - - if not is_tool_call_block: - return DeltaMessage(content=delta_text) - - try: - # Initialize streaming state if not exists - if not hasattr(self, "streaming_state"): - self.streaming_state = { - "current_tool_index": -1, - "tool_ids": [], - "sent_tools": [], # Track complete state of each tool - } - - # Try parsing as JSON to check for complete tool calls - try: - # Use preprocessed tool calls if available - tool_calls_text = ( - preprocessed_tool_calls if preprocessed_tool_calls else current_text - ) - parsed_tools = json.loads(tool_calls_text) - if isinstance(parsed_tools, list): - # Update our tool array for next time - self.prev_tool_call_arr = parsed_tools - except json.JSONDecodeError: - # Not complete JSON yet, use regex for partial parsing - pass - - # Check for test-specific state setup (current_tools_sent) - # This handles the case where tests manually set current_tools_sent - if ( - hasattr(self, "current_tools_sent") # type: ignore - and len(self.current_tools_sent) > 0 - ): - # If current_tools_sent is set to [False], it means the test wants us to send the name - if ( - len(self.current_tools_sent) == 1 - and self.current_tools_sent[0] is False - ): - # Extract the function name using regex - name_pattern = r'"name"\s*:\s*"([^"]+)"' - name_match = re.search(name_pattern, current_text) - if name_match: - function_name = name_match.group(1) - - # The test expects us to send just the name first - tool_id = make_tool_call_id() - delta = DeltaMessage( - tool_calls=[ - DeltaToolCall( - index=0, - type="function", - id=tool_id, - function=DeltaFunctionCall( - name=function_name - ).model_dump(exclude_none=True), # type: ignore - ) - ] - ) - # Update state to reflect that we've sent the name - self.current_tools_sent = [True] - self.current_tool_id = 0 - self.streaming_state["current_tool_index"] = 0 - if len(self.streaming_state["sent_tools"]) == 0: - self.streaming_state["sent_tools"].append( - { - "sent_name": True, - "sent_arguments_prefix": False, - "sent_arguments": "", - } - ) - else: - self.streaming_state["sent_tools"][0]["sent_name"] = True - self.current_tool_name_sent = True - return delta - - # Use regex to identify tool calls in the output - # Use preprocessed tool calls text for better parsing, but also try to extract from incomplete JSON blocks - search_text = ( - preprocessed_tool_calls if preprocessed_tool_calls else current_text - ) - - # For JSON code blocks that aren't complete yet, try to extract the JSON content - if not preprocessed_tool_calls and has_potential_json_block: - # Try to extract the JSON array from within the code block - json_match = re.search( - r"```(?:json)?\s*([\s\S]*?)(?:```|$)", current_text - ) - if json_match: - potential_json = json_match.group(1).strip() - # Use this as search text even if it's incomplete - if potential_json.startswith("[") and ( - '"name"' in potential_json and '"arguments"' in potential_json - ): - search_text = potential_json - - # Try to find complete tool names first - name_pattern = r'"name"\s*:\s*"([^"]+)"' - name_matches = list(re.finditer(name_pattern, search_text)) - tool_count = len(name_matches) - - # If no complete tool names found, check for partial tool names - if tool_count == 0: - # Check if we're in the middle of parsing a tool name - partial_name_pattern = r'"name"\s*:\s*"([^"]*)' - partial_matches = list(re.finditer(partial_name_pattern, search_text)) - if partial_matches: - # We have a partial tool name - not ready to emit yet - return None - else: - # No tools found at all - return None - - # Ensure our state arrays are large enough - while len(self.streaming_state["sent_tools"]) < tool_count: - self.streaming_state["sent_tools"].append( - { - "sent_name": False, - "sent_arguments_prefix": False, - "sent_arguments": "", - } - ) - - while len(self.streaming_state["tool_ids"]) < tool_count: - self.streaming_state["tool_ids"].append(None) - - # Determine if we need to move to a new tool - current_idx = self.streaming_state["current_tool_index"] - - # If we haven't processed any tool yet or current tool is complete, move to next - if current_idx == -1 or current_idx < tool_count - 1: - next_idx = current_idx + 1 - - # If tool at next_idx has not been sent yet - if ( - next_idx < tool_count - and not self.streaming_state["sent_tools"][next_idx]["sent_name"] - ): - # Update indexes - self.streaming_state["current_tool_index"] = next_idx - self.current_tool_id = next_idx # For backward compatibility - current_idx = next_idx - - # Extract the tool name - tool_name = name_matches[current_idx].group(1) - - # Generate ID and send tool name - tool_id = f"call_{current_idx}_{random_uuid()}" - self.streaming_state["tool_ids"][current_idx] = tool_id - - delta = DeltaMessage( - tool_calls=[ - DeltaToolCall( - index=current_idx, - type="function", - id=tool_id, - function=DeltaFunctionCall(name=tool_name).model_dump( - exclude_none=True - ), # type: ignore - ) - ] - ) - self.streaming_state["sent_tools"][current_idx]["sent_name"] = True - self.current_tool_name_sent = True # For backward compatibility - - # Keep track of streamed args for backward compatibility - while len(self.streamed_args) <= current_idx: - self.streamed_args.append("") - - return delta - - # Process arguments for the current tool - if current_idx >= 0 and current_idx < tool_count: - # Support both regular and empty argument objects - # First, check for the empty arguments case: "arguments": {} - empty_args_pattern = ( - r'"name"\s*:\s*"[^"]+"\s*,\s*"arguments"\s*:\s*\{\s*\}' - ) - empty_args_match = re.search(empty_args_pattern, search_text) - - # Check if this tool has empty arguments - if empty_args_match and empty_args_match.start() > 0: - # Find which tool this empty arguments belongs to - empty_args_tool_idx = 0 - for i in range(tool_count): - if i == current_idx: - # If this is our current tool and it has empty arguments - if not self.streaming_state["sent_tools"][current_idx][ - "sent_arguments_prefix" - ]: - # Send empty object - self.streaming_state["sent_tools"][current_idx][ - "sent_arguments_prefix" - ] = True - self.streaming_state["sent_tools"][current_idx][ - "sent_arguments" - ] = "{}" - - # Update streamed_args for backward compatibility - while len(self.streamed_args) <= current_idx: - self.streamed_args.append("") - self.streamed_args[current_idx] += "{}" - - delta = DeltaMessage( - tool_calls=[ - DeltaToolCall( - index=current_idx, - function=DeltaFunctionCall( - arguments="{}" - ).model_dump(exclude_none=True), # type: ignore - ) - ] - ) - - # Move to next tool if available - if current_idx < tool_count - 1: - self.streaming_state["current_tool_index"] += 1 - self.current_tool_id = self.streaming_state[ - "current_tool_index" - ] - - return delta - - # Extract arguments for current tool using regex for non-empty arguments - args_pattern = r'"name"\s*:\s*"[^"]+"\s*,\s*"arguments"\s*:\s*(\{(?:[^{}]|(?:\{[^{}]*\}))*\})' - args_matches = list(re.finditer(args_pattern, search_text)) - - if current_idx < len(args_matches): - args_text = args_matches[current_idx].group(1) - - # Handle transition between tools - is_last_tool = current_idx == tool_count - 1 - - # For multiple tools, extract only the arguments for the current tool - if tool_count > 1: - # Parse the entire JSON structure to properly extract arguments for each tool - try: - parsed_tools = json.loads(search_text) - if isinstance(parsed_tools, list) and current_idx < len( - parsed_tools - ): - current_tool = parsed_tools[current_idx] - if isinstance(current_tool.get("arguments"), dict): - args_text = json.dumps( - current_tool["arguments"], ensure_ascii=False - ) - else: - args_text = str(current_tool.get("arguments", "{}")) - except (json.JSONDecodeError, KeyError, IndexError): - # Fallback to regex-based extraction - pass - - # If arguments haven't been sent yet - sent_args = self.streaming_state["sent_tools"][current_idx][ - "sent_arguments" - ] - - # If we haven't sent the opening bracket yet - if not self.streaming_state["sent_tools"][current_idx][ - "sent_arguments_prefix" - ] and args_text.startswith("{"): - self.streaming_state["sent_tools"][current_idx][ - "sent_arguments_prefix" - ] = True - self.streaming_state["sent_tools"][current_idx][ - "sent_arguments" - ] = "{" - - # Update streamed_args for backward compatibility - while len(self.streamed_args) <= current_idx: - self.streamed_args.append("") - self.streamed_args[current_idx] += "{" - - delta = DeltaMessage( - tool_calls=[ - DeltaToolCall( - index=current_idx, - function=DeltaFunctionCall( - arguments="{" - ).model_dump(exclude_none=True), # type: ignore - ) - ] - ) - return delta - - # If we need to send more arguments - if args_text.startswith(sent_args): - # Calculate what part of arguments we need to send - args_diff = args_text[len(sent_args) :] - - if args_diff: - # Update our state - self.streaming_state["sent_tools"][current_idx][ - "sent_arguments" - ] = args_text - - # Update streamed_args for backward compatibility - while len(self.streamed_args) <= current_idx: - self.streamed_args.append("") - self.streamed_args[current_idx] += args_diff - - delta = DeltaMessage( - tool_calls=[ - DeltaToolCall( - index=current_idx, - function=DeltaFunctionCall( - arguments=args_diff - ).model_dump(exclude_none=True), # type: ignore - ) - ] - ) - return delta - - # If the tool's arguments are complete, check if we need to move to the next tool - if args_text.endswith("}") and args_text == sent_args: - # This tool is complete, move to the next one in the next iteration - if current_idx < tool_count - 1: - self.streaming_state["current_tool_index"] += 1 - self.current_tool_id = self.streaming_state[ - "current_tool_index" - ] # For compatibility - - # If we got here, we couldn't determine what to stream next - return None - - except Exception as e: - logger.exception(f"Error in streaming tool calls: {e}") - # If we encounter an error, just return the delta text as regular content - return DeltaMessage(content=delta_text) diff --git a/vllm/transformers_utils/configs/__init__.py b/vllm/transformers_utils/configs/__init__.py index 68f82ead6b..2233fb95bd 100644 --- a/vllm/transformers_utils/configs/__init__.py +++ b/vllm/transformers_utils/configs/__init__.py @@ -14,71 +14,11 @@ from __future__ import annotations import importlib _CLASS_TO_MODULE: dict[str, str] = { - "BagelConfig": "vllm.transformers_utils.configs.bagel", - "BailingMoeV3TextConfig": "vllm.transformers_utils.configs.bailing_moe_v3_vl", - "BailingMoeV3VisionConfig": "vllm.transformers_utils.configs.bailing_moe_v3_vl", - "BailingMoeV3VLConfig": "vllm.transformers_utils.configs.bailing_moe_v3_vl", - "ChatGLMConfig": "vllm.transformers_utils.configs.chatglm", - "ColModernVBertConfig": "vllm.transformers_utils.configs.colmodernvbert", - "ColPaliConfig": "vllm.transformers_utils.configs.colpali", - "ColQwen3Config": "vllm.transformers_utils.configs.colqwen3", - "OpsColQwen3Config": "vllm.transformers_utils.configs.colqwen3", - "Qwen3VLNemotronEmbedConfig": "vllm.transformers_utils.configs.colqwen3", - "Cosmos3EdgeConfig": "vllm.transformers_utils.configs.cosmos3_edge", - "Cosmos3EdgeProjectorConfig": "vllm.transformers_utils.configs.cosmos3_edge", - "Cosmos3EdgeTextConfig": "vllm.transformers_utils.configs.cosmos3_edge", - "Cosmos3EdgeVisionConfig": "vllm.transformers_utils.configs.cosmos3_edge", - "DeepseekVLV2Config": "vllm.transformers_utils.configs.deepseek_vl2", - "DeepseekV41Config": "vllm.transformers_utils.configs.deepseek_v41", - "Dots3NoteConfig": "vllm.transformers_utils.configs.dots3_note", - "K3DSparkConfig": "vllm.transformers_utils.configs.k3_dspark", - "DotsOCRConfig": "vllm.transformers_utils.configs.dotsocr", "EAGLEConfig": "vllm.transformers_utils.configs.eagle", - "FunAudioChatConfig": "vllm.transformers_utils.configs.funaudiochat", - "FunAudioChatAudioEncoderConfig": "vllm.transformers_utils.configs.funaudiochat", - "HYV4Config": "vllm.transformers_utils.configs.hy_v4", - "IsaacConfig": "vllm.transformers_utils.configs.isaac", # RWConfig is for the original tiiuae/falcon-40b(-instruct) and # tiiuae/falcon-7b(-instruct) models. Newer Falcon models will use the # `FalconConfig` class from the official HuggingFace transformers library. - "RWConfig": "vllm.transformers_utils.configs.falcon", - "MiDashengLMConfig": "vllm.transformers_utils.configs.midashenglm", - "MiniMaxM3Config": "vllm.transformers_utils.configs.minimax_m3", - "MiniMaxM3MTPConfig": "vllm.transformers_utils.configs.minimax_m3", - "MiniMaxM3TextConfig": "vllm.transformers_utils.configs.minimax_m3", - "Moondream3Config": "vllm.transformers_utils.configs.moondream3", - "Moondream3TextConfig": "vllm.transformers_utils.configs.moondream3", - "Moondream3VisionConfig": "vllm.transformers_utils.configs.moondream3", - "MossTranscribeDiarizeConfig": ( - "vllm.transformers_utils.configs.moss_transcribe_diarize" - ), - "MoonViTConfig": "vllm.transformers_utils.configs.moonvit", - "KimiLinearConfig": "vllm.transformers_utils.configs.kimi_linear", - "KimiVLConfig": "vllm.transformers_utils.configs.kimi_vl", - "MuseGlimmerConfig": "vllm.transformers_utils.configs.muse_glimmer", - "MuseGlimmerTextConfig": "vllm.transformers_utils.configs.muse_glimmer", - "MuseGlimmerVisionConfig": "vllm.transformers_utils.configs.muse_glimmer", - "MuseGlimmerAssistantConfig": "vllm.transformers_utils.configs.muse_glimmer", - "KimiK3Config": "vllm.transformers_utils.configs.kimi_k3", - "KimiK3VisionConfig": "vllm.transformers_utils.configs.kimi_k3", - "OpenVLAConfig": "vllm.transformers_utils.configs.openvla", - "OvisConfig": "vllm.transformers_utils.configs.ovis", - "PixelShuffleSiglip2VisionConfig": "vllm.transformers_utils.configs.isaac", - "RadioConfig": "vllm.transformers_utils.configs.radio", - "SpeculatorsConfig": "vllm.transformers_utils.configs.speculators", - "UltravoxConfig": "vllm.transformers_utils.configs.ultravox", - "UnlimitedOCRConfig": "vllm.transformers_utils.configs.unlimited_ocr", - "Step3VLConfig": "vllm.transformers_utils.configs.step3_vl", - "Step3VisionEncoderConfig": "vllm.transformers_utils.configs.step3_vl", - "Step3TextConfig": "vllm.transformers_utils.configs.step3_vl", - "Step3p5Config": "vllm.transformers_utils.configs.step3p5", - "QianfanOCRConfig": "vllm.transformers_utils.configs.qianfan_ocr", - "QianfanOCRVisionConfig": "vllm.transformers_utils.configs.qianfan_ocr", - "Qwen3ASRConfig": "vllm.transformers_utils.configs.qwen3_asr", - "InklingModelConfig": "vllm.models.inkling.configs", - "InklingAudioConfig": "vllm.models.inkling.configs", - "InklingVisionConfig": "vllm.models.inkling.configs", - "InklingMMConfig": "vllm.models.inkling.configs", + "MossTranscribeDiarizeConfig": (), # Upstream Transformers classes registered in _CONFIG_REGISTRY "DeepseekV3Config": "transformers", "HyperCLOVAXConfig": "transformers", diff --git a/vllm/transformers_utils/configs/bailing_moe_v3_vl.py b/vllm/transformers_utils/configs/bailing_moe_v3_vl.py deleted file mode 100644 index e009a4db14..0000000000 --- a/vllm/transformers_utils/configs/bailing_moe_v3_vl.py +++ /dev/null @@ -1,345 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -# Copyright 2025 Antgroup and The HuggingFace Inc. team. All rights reserved. -# Adapted from -# https://huggingface.co/inclusionAI/Ling-3.0-flash/blob/51fd444268df5267074ef0b289de0e1c4cb9b381/configuration_bailing_moe_v3.py -"""Bailing V3 vision-language model configuration.""" - -from __future__ import annotations - -from typing import Any - -from transformers.configuration_utils import PreTrainedConfig - - -def _normalize_mrope_parameters( - config: PreTrainedConfig, - mrope_section: list[int], -) -> None: - """Add the top-level Bailing M-RoPE split to a text config.""" - rope_parameters = dict(getattr(config, "rope_parameters", None) or {}) - text_mrope_section = rope_parameters.get("mrope_section") - if text_mrope_section is not None and list(text_mrope_section) != mrope_section: - raise ValueError( - "Bailing V3 VL has conflicting M-RoPE sections: " - f"text={text_mrope_section}, top-level={mrope_section}" - ) - - if "rope_type" not in rope_parameters and "type" not in rope_parameters: - rope_parameters["rope_type"] = "default" - rope_parameters.setdefault("rope_theta", getattr(config, "rope_theta", 10_000)) - rope_parameters["mrope_section"] = list(mrope_section) - config.rope_parameters = rope_parameters - - -def _build_layer_types( - num_hidden_layers: int, - layer_group_size: int, -) -> list[str]: - if layer_group_size <= 0: - raise ValueError( - f"Bailing V3 VL layer_group_size must be positive, got {layer_group_size}" - ) - - grouped_layers = num_hidden_layers // layer_group_size * layer_group_size - return [ - "linear_attention" - if ((layer_idx + 1) % layer_group_size != 0 and layer_idx < grouped_layers) - else "full_attention" - for layer_idx in range(num_hidden_layers) - ] - - -class BailingMoeV3TextConfig(PreTrainedConfig): - model_type = "bailing_hybrid" - base_config_key = "text_config" - keys_to_ignore_at_inference = ["past_key_values"] - ignore_keys_at_rope_validation = {"mrope_section"} - - def __init__( - self, - *, - vocab_size: int = 157184, - hidden_size: int = 2048, - intermediate_size: int = 5120, - num_hidden_layers: int = 20, - num_attention_heads: int = 16, - num_key_value_heads: int = 4, - hidden_act: str = "silu", - use_qkv_bias: bool = False, - use_bias: bool = False, - rms_norm_eps: float = 1e-6, - tie_word_embeddings: bool = False, - embedding_dropout: float = 0.0, - attention_dropout: float = 0.0, - output_dropout: float = 0.0, - initializer_range: float = 0.02, - max_position_embeddings: int = 32768, - rope_theta: float = 600000.0, - rope_parameters: dict[str, Any] | None = None, - rope_scaling: dict[str, Any] | None = None, - partial_rotary_factor: float | None = None, - mrope_section: list[int] | tuple[int, ...] | None = None, - use_cache: bool = True, - max_window_layers: int = 20, - pad_token_id: int = 156892, - eos_token_id: int = 156892, - num_experts: int = 256, - num_shared_experts: int = 1, - num_experts_per_tok: int = 8, - n_group: int = 8, - topk_group: int = 4, - moe_intermediate_size: int = 512, - moe_shared_expert_intermediate_size: int = 512, - first_k_dense_replace: int = 1, - head_dim: int | None = 128, - output_router_logits: bool = False, - use_qk_norm: bool = True, - num_nextn_predict_layers: int = 0, - mtp_loss_scaling_factor: float = 0, - moe_router_enable_expert_bias: bool = True, - routed_scaling_factor: float = 1.0, - layer_group_size: int = 5, - layer_types: list[str] | None = None, - layers_block_type: list[str] | None = None, - kv_lora_rank: int = 512, - q_lora_rank: int | None = None, - qk_rope_head_dim: int = 64, - v_head_dim: int = 128, - qk_nope_head_dim: int = 128, - rope_interleave: bool = True, - score_function: str = "sigmoid", - scoring_func: str = "sigmoid", - seq_aux: bool = True, - topk_method: str = "noaux_tc", - router_dtype: str = "fp32", - gated_attention_proj_granularity_type: str | None = None, - no_kda_lora: bool = False, - kda_safe_gate: bool = False, - kda_lower_bound: float | None = None, - short_conv_kernel_size: int = 4, - architectures: list[str] | None = None, - **kwargs: Any, - ) -> None: - kwargs.pop("model_type", None) - if architectures is None: - architectures = ["BailingMoeV3ForCausalLM"] - - if rope_parameters is not None: - self.rope_parameters = dict(rope_parameters) - elif rope_scaling is not None: - self.rope_parameters = dict(rope_scaling) - else: - self.rope_parameters = {} - if ( - "rope_type" not in self.rope_parameters - and "type" not in self.rope_parameters - ): - self.rope_parameters["rope_type"] = "default" - self.rope_parameters.setdefault("rope_theta", rope_theta) - if mrope_section is not None: - _normalize_mrope_parameters(self, list(mrope_section)) - - self.vocab_size = vocab_size - self.hidden_size = hidden_size - self.intermediate_size = intermediate_size - self.num_hidden_layers = num_hidden_layers - self.num_attention_heads = num_attention_heads - self.num_key_value_heads = num_key_value_heads - self.hidden_act = hidden_act - self.use_qkv_bias = use_qkv_bias - self.use_bias = use_bias - self.rms_norm_eps = rms_norm_eps - self.embedding_dropout = embedding_dropout - self.attention_dropout = attention_dropout - self.output_dropout = output_dropout - self.initializer_range = initializer_range - self.max_position_embeddings = max_position_embeddings - self.rope_theta = rope_theta - self.partial_rotary_factor = partial_rotary_factor - self.use_cache = use_cache - self.max_window_layers = max_window_layers - self.num_experts = num_experts - self.num_shared_experts = num_shared_experts - self.num_experts_per_tok = num_experts_per_tok - self.n_group = n_group - self.topk_group = topk_group - self.moe_intermediate_size = moe_intermediate_size - self.moe_shared_expert_intermediate_size = moe_shared_expert_intermediate_size - self.first_k_dense_replace = first_k_dense_replace - self.head_dim = head_dim or hidden_size // num_attention_heads - self.output_router_logits = output_router_logits - self.use_qk_norm = use_qk_norm - self.num_nextn_predict_layers = num_nextn_predict_layers - self.mtp_loss_scaling_factor = mtp_loss_scaling_factor - self.moe_router_enable_expert_bias = moe_router_enable_expert_bias - self.routed_scaling_factor = routed_scaling_factor - self.layer_group_size = layer_group_size - self.kv_lora_rank = kv_lora_rank - self.q_lora_rank = q_lora_rank - self.qk_rope_head_dim = qk_rope_head_dim - self.v_head_dim = v_head_dim - self.qk_nope_head_dim = qk_nope_head_dim - self.qk_head_dim = qk_nope_head_dim + qk_rope_head_dim - self.rope_interleave = rope_interleave - self.score_function = score_function - self.scoring_func = scoring_func - self.seq_aux = seq_aux - self.topk_method = topk_method - self.router_dtype = router_dtype - self.gated_attention_proj_granularity_type = ( - gated_attention_proj_granularity_type - ) - self.no_kda_lora = no_kda_lora - self.kda_safe_gate = kda_safe_gate - self.kda_lower_bound = kda_lower_bound - self.short_conv_kernel_size = short_conv_kernel_size - - expected_layer_types = _build_layer_types( - num_hidden_layers, - layer_group_size, - ) - if layers_block_type is not None: - legacy_type_mapping = { - "attention": "full_attention", - "mamba": "linear_attention", - "conv": "linear_attention", - } - normalized_block_types = [ - legacy_type_mapping.get(layer_type, layer_type) - for layer_type in layers_block_type - ] - if layer_types is not None and layer_types != normalized_block_types: - raise ValueError( - "Bailing V3 VL has conflicting layer_types and layers_block_type" - ) - layer_types = normalized_block_types - if layer_types is not None and layer_types != expected_layer_types: - raise ValueError( - "Bailing V3 VL layer_types must match the layer_group_size schedule" - ) - self.layer_types = expected_layer_types - - super().__init__( - architectures=architectures, - pad_token_id=pad_token_id, - eos_token_id=eos_token_id, - tie_word_embeddings=tie_word_embeddings, - **kwargs, - ) - self.validate_layer_type() - - -class BailingMoeV3VisionConfig(PreTrainedConfig): - model_type = "qwen3_moe_vit" - base_config_key = "vision_config" - - def __init__( - self, - *, - depth: int = 27, - hidden_size: int = 1152, - hidden_act: str = "gelu_pytorch_tanh", - intermediate_size: int = 4304, - num_heads: int = 16, - in_channels: int = 3, - patch_size: int = 16, - spatial_merge_size: int = 2, - temporal_patch_size: int = 2, - out_hidden_size: int = 3584, - num_position_embeddings: int = 2304, - deepstack_visual_indexes: list[int] | None = None, - initializer_range: float = 0.02, - **kwargs: Any, - ) -> None: - kwargs.pop("model_type", None) - super().__init__(**kwargs) - self.depth = depth - self.hidden_size = hidden_size - self.hidden_act = hidden_act - self.intermediate_size = intermediate_size - self.num_heads = num_heads - self.in_channels = in_channels - self.patch_size = patch_size - self.spatial_merge_size = spatial_merge_size - self.temporal_patch_size = temporal_patch_size - self.out_hidden_size = out_hidden_size - self.num_position_embeddings = num_position_embeddings - self.deepstack_visual_indexes = ( - [8, 16, 24] - if deepstack_visual_indexes is None - else deepstack_visual_indexes - ) - self.initializer_range = initializer_range - - -class BailingMoeV3VLConfig(PreTrainedConfig): - model_type = "bailing_moe_v3_vl" - keys_to_ignore_at_inference = ["past_key_values"] - sub_configs = { - "text_config": BailingMoeV3TextConfig, - "vision_config": BailingMoeV3VisionConfig, - } - - def __init__( - self, - *, - text_config: BailingMoeV3TextConfig | dict[str, Any] | None = None, - vision_config: BailingMoeV3VisionConfig | dict[str, Any] | None = None, - image_token_id: int = 151655, - video_token_id: int = 151656, - vision_start_token_id: int = 151652, - vision_end_token_id: int = 151653, - tie_word_embeddings: bool = False, - mrope_section: list[int] | tuple[int, ...] | None = None, - architectures: list[str] | None = None, - **kwargs: Any, - ) -> None: - kwargs.pop("model_type", None) - if mrope_section is None: - mrope_section = [12, 10, 10] - self.mrope_section = list(mrope_section) - - if text_config is None: - text_config = BailingMoeV3TextConfig() - elif isinstance(text_config, dict): - text_config = dict(text_config) - nested_mrope_section = text_config.pop("mrope_section", None) - if ( - nested_mrope_section is not None - and list(nested_mrope_section) != self.mrope_section - ): - raise ValueError( - "Bailing V3 VL has conflicting M-RoPE sections: " - f"text={nested_mrope_section}, " - f"top-level={self.mrope_section}" - ) - text_config = BailingMoeV3TextConfig(**text_config) - self.text_config = text_config - _normalize_mrope_parameters(self.text_config, self.mrope_section) - - if vision_config is None: - vision_config = BailingMoeV3VisionConfig() - elif isinstance(vision_config, dict): - vision_config = BailingMoeV3VisionConfig(**vision_config) - self.vision_config = vision_config - - self.image_token_id = image_token_id - self.video_token_id = video_token_id - self.vision_start_token_id = vision_start_token_id - self.vision_end_token_id = vision_end_token_id - if architectures is None: - architectures = ["BailingMoeV3VLForConditionalGeneration"] - - super().__init__( - architectures=architectures, - tie_word_embeddings=tie_word_embeddings, - **kwargs, - ) - - -__all__ = [ - "BailingMoeV3TextConfig", - "BailingMoeV3VisionConfig", - "BailingMoeV3VLConfig", -] diff --git a/vllm/transformers_utils/configs/chatglm.py b/vllm/transformers_utils/configs/chatglm.py deleted file mode 100644 index 3908df740a..0000000000 --- a/vllm/transformers_utils/configs/chatglm.py +++ /dev/null @@ -1,75 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -# Adapted from -# https://github.com/zai-org/ChatGLM2-6B -from transformers import PreTrainedConfig - - -class ChatGLMConfig(PreTrainedConfig): - model_type = "chatglm" - attribute_map = { - "num_hidden_layers": "num_layers", - "n_head_kv": "multi_query_group_num", - } - - def __init__( - self, - num_layers=28, - padded_vocab_size=65024, - hidden_size=4096, - ffn_hidden_size=13696, - kv_channels=128, - num_attention_heads=32, - seq_length=2048, - hidden_dropout=0.0, - attention_dropout=0.0, - layernorm_epsilon=1e-5, - rmsnorm=True, - apply_residual_connection_post_layernorm=False, - post_layer_norm=True, - add_bias_linear=False, - add_qkv_bias=False, - interleaved_qkv=False, - bias_dropout_fusion=True, - multi_query_attention=False, - multi_query_group_num=1, - apply_query_key_layer_scaling=True, - attention_softmax_in_fp32=True, - fp32_residual_connection=False, - quantization_bit=0, - pre_seq_len=None, - prefix_projection=False, - **kwargs, - ): - self.num_layers = num_layers - self.vocab_size = padded_vocab_size - self.padded_vocab_size = padded_vocab_size - self.hidden_size = hidden_size - self.ffn_hidden_size = ffn_hidden_size - self.kv_channels = kv_channels - self.num_attention_heads = num_attention_heads - self.seq_length = seq_length - # It is to be compatible with long lora. - self.max_position_embeddings = seq_length - self.hidden_dropout = hidden_dropout - self.attention_dropout = attention_dropout - self.layernorm_epsilon = layernorm_epsilon - self.rmsnorm = rmsnorm - self.apply_residual_connection_post_layernorm = ( - apply_residual_connection_post_layernorm - ) - self.post_layer_norm = post_layer_norm - self.add_bias_linear = add_bias_linear - self.add_qkv_bias = add_qkv_bias - self.bias_dropout_fusion = bias_dropout_fusion - self.multi_query_attention = multi_query_attention - self.multi_query_group_num = multi_query_group_num - self.apply_query_key_layer_scaling = apply_query_key_layer_scaling - self.attention_softmax_in_fp32 = attention_softmax_in_fp32 - self.fp32_residual_connection = fp32_residual_connection - self.quantization_bit = quantization_bit - self.pre_seq_len = pre_seq_len - self.prefix_projection = prefix_projection - self.interleaved_qkv = interleaved_qkv - super().__init__(**kwargs) diff --git a/vllm/transformers_utils/configs/colmodernvbert.py b/vllm/transformers_utils/configs/colmodernvbert.py deleted file mode 100644 index a830541757..0000000000 --- a/vllm/transformers_utils/configs/colmodernvbert.py +++ /dev/null @@ -1,63 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Configuration for ColModernVBERT visual document retrieval model. - -ColModernVBERT combines SigLIP vision encoder + ModernBERT text encoder -with a pixel shuffle connector and ColBERT-style 128-dim per-token embeddings. - -Reference: https://huggingface.co/ModernVBERT/colmodernvbert-merged -""" - -from transformers import ModernBertConfig, PreTrainedConfig, SiglipVisionConfig - - -class ColModernVBertConfig(PreTrainedConfig): - model_type = "colmodernvbert" - - def __init__( - self, - embedding_dim: int = 128, - image_token_id: int = 50407, - pixel_shuffle_factor: int = 4, - text_config: dict | None = None, - vision_config: dict | None = None, - **kwargs, - ): - self.embedding_dim = embedding_dim - self.image_token_id = image_token_id - self.pixel_shuffle_factor = pixel_shuffle_factor - - text_config = text_config or {} - self.hidden_size = text_config.get("hidden_size", 768) - - self.text_config = ModernBertConfig( - vocab_size=text_config.get("vocab_size", 50408), - hidden_size=text_config.get("hidden_size", 768), - intermediate_size=text_config.get("intermediate_size", 1152), - num_hidden_layers=text_config.get("num_hidden_layers", 22), - num_attention_heads=text_config.get("num_attention_heads", 12), - mlp_bias=text_config.get("mlp_bias", False), - max_position_embeddings=text_config.get("max_position_embeddings", 8192), - ) - - vision_config = vision_config or {} - self.vision_config = SiglipVisionConfig( - hidden_size=vision_config.get("hidden_size", 768), - image_size=vision_config.get("image_size", 512), - patch_size=vision_config.get("patch_size", 16), - num_hidden_layers=vision_config.get("num_hidden_layers", 12), - intermediate_size=vision_config.get("intermediate_size", 3072), - num_attention_heads=vision_config.get("num_attention_heads", 12), - ) - - # Ensure architectures is set so vLLM routes to our model class - kwargs.setdefault("architectures", ["ColModernVBertForRetrieval"]) - super().__init__(**kwargs) - - @property - def image_seq_len(self) -> int: - ps = self.vision_config.image_size // self.vision_config.patch_size - return (ps * ps) // (self.pixel_shuffle_factor**2) - - def get_text_config(self, **kwargs): - return self.text_config diff --git a/vllm/transformers_utils/configs/colpali.py b/vllm/transformers_utils/configs/colpali.py deleted file mode 100644 index a4e0a83e29..0000000000 --- a/vllm/transformers_utils/configs/colpali.py +++ /dev/null @@ -1,56 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""ColPali configuration that extends PaliGemmaConfig with embedding projection -fields. This allows ColPali models to be loaded without trust_remote_code -by mapping their custom model_type (colpali) to a standard config class -that vLLM understands. - -Supported model_types: -- colpali (vidore/colpali-v1.3-hf) -""" - -from transformers import PaliGemmaConfig - - -class ColPaliConfig(PaliGemmaConfig): - """Configuration class for ColPali models. - - Extends PaliGemmaConfig with additional fields used by ColPali variants - for the embedding projection layer. - """ - - model_type = "colpali" - - def __init__( - self, - embedding_dim: int | None = None, - embed_dim: int | None = None, - dim: int | None = None, - colbert_dim: int | None = None, - pooling: str | None = None, - vlm_config: dict | None = None, - **kwargs, - ): - # Store embedding projection config fields - self.embedding_dim = embedding_dim - self.embed_dim = embed_dim - self.dim = dim - self.colbert_dim = colbert_dim - self.pooling = pooling - - # The HF checkpoint nests PaliGemma config inside "vlm_config". - # Flatten it so PaliGemmaConfig receives vision_config, text_config, - # image_token_index, etc. directly. - # Use setdefault to avoid overwriting keys already set (e.g. - # model_type="colpali" would be clobbered by "paligemma" from - # vlm_config). - if vlm_config is not None: - vlm_dict = ( - vlm_config if isinstance(vlm_config, dict) else vlm_config.to_dict() - ) - _conflicting = {"model_type", "_name_or_path"} - for key, value in vlm_dict.items(): - if key not in _conflicting: - kwargs.setdefault(key, value) - - super().__init__(**kwargs) diff --git a/vllm/transformers_utils/configs/cosmos3_edge.py b/vllm/transformers_utils/configs/cosmos3_edge.py deleted file mode 100644 index ddf17933fd..0000000000 --- a/vllm/transformers_utils/configs/cosmos3_edge.py +++ /dev/null @@ -1,147 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from transformers import NemotronHConfig -from transformers.configuration_utils import PreTrainedConfig -from transformers.models.siglip2.configuration_siglip2 import Siglip2VisionConfig - - -def _normalize_mrope_parameters(config: NemotronHConfig) -> None: - """Normalize the checkpoint's mRoPE configuration for vLLM.""" - rope_parameters = dict(getattr(config, "rope_parameters", None) or {}) - mrope_section = rope_parameters.get("mrope_section") - if mrope_section is None: - return - - rope_parameters["mrope_section"] = list(mrope_section) - rope_parameters.setdefault("mrope_interleaved", True) - config.rope_parameters = rope_parameters - - -class Cosmos3EdgeTextConfig(NemotronHConfig): - """Dense Cosmos3 Edge text config backed by Nemotron-H layers.""" - - model_type = "cosmos3_edge_text" - has_no_defaults_at_init = True - ignore_keys_at_rope_validation = {"mrope_section", "mrope_interleaved"} - - def __init__( - self, - num_hidden_layers: int = 28, - hidden_act: str = "relu2", - rms_norm_eps: float = 1e-5, - layers_block_type: list[str] | None = None, - **kwargs, - ) -> None: - # Each decoder block is an attention layer followed by an MLP layer - if layers_block_type is None: - layers_block_type = ["full_attention", "mlp"] * num_hidden_layers - super().__init__( - layers_block_type=layers_block_type, - mlp_hidden_act=hidden_act, - layer_norm_epsilon=rms_norm_eps, - **kwargs, - ) - - def to_dict(self) -> dict: - config_dict = super().to_dict() - config_dict["num_hidden_layers"] = self.num_hidden_layers // 2 - config_dict["hidden_act"] = config_dict.pop("mlp_hidden_act") - config_dict["rms_norm_eps"] = config_dict.pop("layer_norm_epsilon") - return config_dict - - -class Cosmos3EdgeProjectorConfig(PreTrainedConfig): - model_type = "cosmos3_edge_projector" - - def __init__( - self, - input_hidden_size: int = 1024, - use_postshuffle_norm: bool = False, - spatial_merge_size: int = 2, - merger_intermediate_size: int = 11520, - out_hidden_size: int = 1024, - **kwargs, - ) -> None: - super().__init__(**kwargs) - self.input_hidden_size = input_hidden_size - self.use_postshuffle_norm = use_postshuffle_norm - self.spatial_merge_size = spatial_merge_size - self.merger_intermediate_size = merger_intermediate_size - self.out_hidden_size = out_hidden_size - - -class Cosmos3EdgeVisionConfig(Siglip2VisionConfig): - model_type = "cosmos3_edge_vision" - - -class Cosmos3EdgeConfig(PreTrainedConfig): - model_type = "cosmos3_edge" - keys_to_ignore_at_inference = ["past_key_values"] - has_no_defaults_at_init = True - sub_configs = { - "vision_config": Cosmos3EdgeVisionConfig, - "projector_config": Cosmos3EdgeProjectorConfig, - "text_config": Cosmos3EdgeTextConfig, - } - - def __init__( - self, - text_config: Cosmos3EdgeTextConfig | dict | None = None, - vision_config: Cosmos3EdgeVisionConfig | dict | None = None, - projector_config: Cosmos3EdgeProjectorConfig | dict | None = None, - image_token_id: int = 19, - video_token_id: int = 18, - vision_start_token_id: int = 20, - vision_end_token_id: int = 21, - tie_word_embeddings: bool = False, - **kwargs, - ) -> None: - if text_config is None: - text_config = {} - if vision_config is None: - vision_config = {} - if projector_config is None: - projector_config = {} - - self.text_config = ( - Cosmos3EdgeTextConfig(**text_config) - if isinstance(text_config, dict) - else text_config - ) - _normalize_mrope_parameters(self.text_config) - - self.vision_config = ( - Cosmos3EdgeVisionConfig(**vision_config) - if isinstance(vision_config, dict) - else vision_config - ) - self.projector_config = ( - Cosmos3EdgeProjectorConfig(**projector_config) - if isinstance(projector_config, dict) - else projector_config - ) - - # Qwen3-VL processing and M-RoPE read these attributes from the - # vision config, while this checkpoint stores them in projector config. - self.vision_config.spatial_merge_size = self.projector_config.spatial_merge_size - self.vision_config.temporal_patch_size = 1 - self.vision_config.out_hidden_size = self.projector_config.out_hidden_size - - self.image_token_id = image_token_id - self.video_token_id = video_token_id - self.vision_start_token_id = vision_start_token_id - self.vision_end_token_id = vision_end_token_id - - super().__init__( - tie_word_embeddings=tie_word_embeddings, - **kwargs, - ) - - -__all__ = [ - "Cosmos3EdgeConfig", - "Cosmos3EdgeProjectorConfig", - "Cosmos3EdgeTextConfig", - "Cosmos3EdgeVisionConfig", -] diff --git a/vllm/transformers_utils/configs/deepseek_v41.py b/vllm/transformers_utils/configs/deepseek_v41.py deleted file mode 100644 index f1ed6f9aa8..0000000000 --- a/vllm/transformers_utils/configs/deepseek_v41.py +++ /dev/null @@ -1,62 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from typing import Any - -from transformers import PreTrainedConfig - - -class DeepseekV41Config(PreTrainedConfig): - """DeepSeek V4.1 config. - - The HF config nests the text model under ``text_config`` and the vision - tower under ``vision_config``. The vLLM model code (ported from - ``deepseek_v4``) reads flat attributes, so both sub-configs are flattened - onto the top level here: text fields are exposed as-is, vision fields - with the ``vision_*`` naming used by ``deepseek_v41.common.vision``. - """ - - model_type = "deepseek_v41" - - def __init__( - self, - text_config: dict[str, Any] | None = None, - vision_config: dict[str, Any] | None = None, - **kwargs, - ): - text_config = dict(text_config or {}) - vision_config = dict(vision_config or {}) - - # ``rope_scaling`` is a property in Transformers v5 (backed by - # ``rope_parameters``); capture it here and restore after - # super().__init__, which re-standardizes rope params. - rope_scaling = text_config.pop("rope_scaling", None) - - for key, value in text_config.items(): - if key == "model_type": - continue - # Don't clobber PreTrainedConfig properties (e.g. is_encoder_decoder). - if isinstance(getattr(type(self), key, None), property): - continue - setattr(self, key, value) - - super().__init__(**kwargs) - if rope_scaling is not None: - self.rope_parameters = rope_scaling - - # The v4.1 quantization_config carries the expert dtype; the model - # code reads it from the top-level hf_config. - quant_cfg = getattr(self, "quantization_config", None) or {} - if not hasattr(self, "expert_dtype") and "expert_dtype" in quant_cfg: - self.expert_dtype = quant_cfg["expert_dtype"] - - vision_n_layers = vision_config.get("num_hidden_layers", 0) - self.vision_n_layers = vision_n_layers - self.vision_dim = vision_config.get("hidden_size", 1024) - self.vision_n_heads = vision_config.get("num_attention_heads", 16) - self.vision_inter_dim = vision_config.get("intermediate_size", 2816) - self.vision_patch_size = vision_config.get("patch_size", 14) - self.vision_rope_theta = vision_config.get("rope_theta", 10000.0) - self.vision_downsample_ratio = vision_config.get("downsample_ratio", 3) - self.vision_max_n_token = vision_config.get("max_image_tokens", 1024) - self.vision_min_pixels = vision_config.get("min_pixels", 295936) - self.vision_max_wh_ratio = vision_config.get("max_wh_ratio") diff --git a/vllm/transformers_utils/configs/deepseek_vl2.py b/vllm/transformers_utils/configs/deepseek_vl2.py deleted file mode 100644 index a819f12f3e..0000000000 --- a/vllm/transformers_utils/configs/deepseek_vl2.py +++ /dev/null @@ -1,126 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -# adapted from https://github.com/deepseek-ai/DeepSeek-VL2/blob/faf18023f24b962b32d9f0a2d89e402a8d383a78/deepseek_vl2/models/modeling_deepseek_vl_v2.py#L115-L268 - -from huggingface_hub.dataclasses import strict -from transformers import DeepseekV2Config, PreTrainedConfig - - -class VisionEncoderConfig(PreTrainedConfig): - model_type: str = "vision" - - model_name: str = "vit_so400m_patch14_siglip_384.webli" - image_size: int = 384 - patch_size: int = 16 - width: int = 1024 - layers: int = 24 - heads: int = 16 - mlp_ratio: int = 4 - global_pool: str = "map" - ignore_head: bool = True - class_token: bool = False - num_classes: int = 0 - use_checkpoint: bool = False - weight_init: str = "skip" - deterministic: bool = False - num_recomputing_layers: int = 0 - - def __init__( - self, - model_name: str = "vit_so400m_patch14_siglip_384.webli", - image_size: int = 384, - patch_size: int = 16, - width: int = 1024, - layers: int = 24, - heads: int = 16, - mlp_ratio: int = 4, - global_pool: str = "map", - ignore_head: bool = True, - class_token: bool = False, - num_classes: int = 0, - use_checkpoint: bool = False, - **kwargs, - ): - self.model_name = model_name - self.image_size = image_size - self.patch_size = patch_size - self.width = width - self.layers = layers - self.heads = heads - self.mlp_ratio = mlp_ratio - self.global_pool = global_pool - self.ignore_head = ignore_head - self.class_token = class_token - self.num_classes = num_classes - self.use_checkpoint = use_checkpoint - - super().__init__(**kwargs) - - -class MlpProjectorConfig(PreTrainedConfig): - model_type = "mlp_projector" - projector_type: str = "downsample_mlp_gelu" - input_dim: int = 1152 - n_embed: int = 2048 - depth: int = 2 - mlp_ratio: int = 1 - downsample_ratio: int = 2 - token_pooling: bool = False - - def __init__( - self, - projector_type: str = "downsample_mlp_gelu", - input_dim: int = 1152, - n_embed: int = 2048, - depth: int = 2, - mlp_ratio: int = 1, - downsample_ratio: int = 2, - **kwargs, - ): - self.projector_type = projector_type - self.input_dim = input_dim - self.n_embed = n_embed - self.depth = depth - self.mlp_ratio = mlp_ratio - self.downsample_ratio = downsample_ratio - - super().__init__(**kwargs) - - -@strict -class DeepseekVLV2TextConfig(DeepseekV2Config): - kv_lora_rank: int | None = DeepseekV2Config.kv_lora_rank - - -class DeepseekVLV2Config(PreTrainedConfig): - model_type = "deepseek_vl_v2" - - tile_tag: str = "2D" - global_view_pos: str = "head" - candidate_resolutions: tuple[tuple[int, int]] = ((384, 384),) - - def __init__( - self, - tile_tag: str = "tile_tag", - global_view_pos: str = "head", - candidate_resolutions: tuple[tuple[int, int]] = ((384, 384),), - **kwargs, - ): - architectures = kwargs.setdefault("architectures", ["DeepseekVLV2ForCausalLM"]) - - self.vision_config = VisionEncoderConfig(**kwargs.pop("vision_config", {})) - self.projector_config = MlpProjectorConfig(**kwargs.pop("projector_config", {})) - self.text_config = DeepseekVLV2TextConfig(**kwargs.pop("language_config", {})) - - self.tile_tag = tile_tag - self.global_view_pos = global_view_pos - self.candidate_resolutions = candidate_resolutions - self.vocab_size = self.text_config.vocab_size - - # update model_type for OCR models - if "DeepseekOCRForCausalLM" in architectures: - kwargs["model_type"] = "deepseek_ocr" - elif "DeepseekOCR2ForCausalLM" in architectures: - kwargs["model_type"] = "deepseek_ocr2" - super().__init__(**kwargs) diff --git a/vllm/transformers_utils/configs/dots3_note.py b/vllm/transformers_utils/configs/dots3_note.py deleted file mode 100644 index 832761544a..0000000000 --- a/vllm/transformers_utils/configs/dots3_note.py +++ /dev/null @@ -1,25 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from transformers import DeepseekV3Config - - -class Dots3NoteConfig(DeepseekV3Config): - """Configuration for the Dots3Note hybrid DSA/SWA model.""" - - model_type = "dots3_note" - - def __init__(self, **kwargs): - # These fields are absent from existing Note config.json files. Do - # not inherit DeepSeek-V3's 8-group/4-group router defaults: Note was - # trained with an ungrouped (1/1) noaux_tc router, as defined by the - # reference Dots3Config in SGLang haimain. A different grouping - # changes the selected experts at every MoE layer. - kwargs.setdefault("n_group", 1) - kwargs.setdefault("topk_group", 1) - # Dots3 projects the indexer RoPE coordinates in adjacent (GPT-J) - # pairs. DeepSeek-V3.2 defaults to the split-half NeoX layout when - # this flag is absent, which rotates different learned coordinates. - kwargs.setdefault("indexer_rope_interleave", True) - kwargs.setdefault("num_nextn_predict_layers", 1) - super().__init__(**kwargs) diff --git a/vllm/transformers_utils/configs/falcon.py b/vllm/transformers_utils/configs/falcon.py deleted file mode 100644 index 7dfe21d0b8..0000000000 --- a/vllm/transformers_utils/configs/falcon.py +++ /dev/null @@ -1,89 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -# Adapted from -# https://huggingface.co/tiiuae/falcon-7b/blob/main/configuration_RW.py -# Copyright 2023 The vLLM team. -# Copyright 2022 the Big Science Workshop and HuggingFace Inc. team. -# All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Falcon configuration.""" - -from transformers.configuration_utils import PreTrainedConfig - - -class RWConfig(PreTrainedConfig): - model_type = "falcon" - keys_to_ignore_at_inference = ["past_key_values"] - attribute_map = { - "num_hidden_layers": "n_layer", - "num_attention_heads": "n_head", - "num_kv_heads": "n_head_kv", - } - - def __init__( - self, - vocab_size=250880, - hidden_size=64, - n_layer=2, - n_head=8, - layer_norm_epsilon=1e-5, - initializer_range=0.02, - use_cache=True, - bos_token_id=1, - eos_token_id=2, - hidden_dropout=0.0, - attention_dropout=0.0, - multi_query=True, - n_head_kv=None, - alibi=False, - bias=False, - parallel_attn=False, - new_decoder_architecture=False, - **kwargs, - ) -> None: - self.vocab_size = vocab_size - # Backward compatibility with n_embed kwarg - n_embed = kwargs.pop("n_embed", None) - self.hidden_size = hidden_size if n_embed is None else n_embed - self.n_layer = n_layer - self.n_head = n_head - self.layer_norm_epsilon = layer_norm_epsilon - self.initializer_range = initializer_range - self.use_cache = use_cache - self.hidden_dropout = hidden_dropout - self.attention_dropout = attention_dropout - - self.bos_token_id = bos_token_id - self.eos_token_id = eos_token_id - self.multi_query = multi_query - self.n_head_kv = 1 if n_head_kv is None else n_head_kv - self.alibi = alibi - self.bias = bias - self.parallel_attn = parallel_attn - self.new_decoder_architecture = new_decoder_architecture - - if self.hidden_size == 8192: - # Hack for falcon-40b - self.new_decoder_architecture = True - - super().__init__(bos_token_id=bos_token_id, eos_token_id=eos_token_id, **kwargs) - - @property - def head_dim(self): - return self.hidden_size // self.n_head - - @property - def rotary(self): - return not self.alibi diff --git a/vllm/transformers_utils/configs/funaudiochat.py b/vllm/transformers_utils/configs/funaudiochat.py deleted file mode 100644 index dd2573aac4..0000000000 --- a/vllm/transformers_utils/configs/funaudiochat.py +++ /dev/null @@ -1,120 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from __future__ import annotations - -from transformers import CONFIG_MAPPING, PreTrainedConfig - -# NOTE: Temporary shim for FunAudioChat checkpoints. -# These checkpoints use `model_type="funaudiochat"`, which is not currently -# recognized by released Transformers, and the public checkpoint does not -# provide an `auto_map` to enable `trust_remote_code=True`. -# Remove this file once Transformers adds native support (or the checkpoint -# provides an `auto_map`) and vLLM can rely on `AutoConfig.from_pretrained()`. - - -class FunAudioChatAudioEncoderConfig(PreTrainedConfig): - model_type = "funaudiochat_audio_encoder" - - def __init__( - self, - _attn_implementation: str | None = None, - num_mel_bins: int = 128, - encoder_layers: int = 32, - encoder_attention_heads: int = 20, - encoder_ffn_dim: int = 5120, - d_model: int = 1280, - dropout: float = 0.0, - attention_dropout: float = 0.0, - activation_function: str = "gelu", - activation_dropout: float = 0.0, - scale_embedding: bool = False, - initializer_range: float = 0.02, - max_source_positions: int = 1500, - n_window: int = 100, - output_dim: int = 3584, - bos_token_id: int | None = None, - codebook_size: int | None = None, - continuous_features_mode: str = "replace", - crq_transformer_config: dict | None = None, - eos_token_id: int | None = None, - group_size: int = 5, - enable_audio_invert_tower: bool = True, - pad_token_id: int | None = None, - **kwargs, - ) -> None: - attn_impl = kwargs.pop("_attn_implementation", None) or _attn_implementation - super().__init__(**kwargs) - # Match HF default for attention implementation selection. - self._attn_implementation = attn_impl or "sdpa" - - self.num_mel_bins = num_mel_bins - self.d_model = d_model - self.encoder_layers = encoder_layers - self.encoder_attention_heads = encoder_attention_heads - self.encoder_ffn_dim = encoder_ffn_dim - self.dropout = dropout - self.attention_dropout = attention_dropout - self.activation_function = activation_function - self.activation_dropout = activation_dropout - self.num_hidden_layers = encoder_layers - self.initializer_range = initializer_range - self.scale_embedding = scale_embedding - self.max_source_positions = max_source_positions - self.n_window = n_window - self.output_dim = output_dim - - self.bos_token_id = bos_token_id - self.codebook_size = codebook_size - self.continuous_features_mode = continuous_features_mode - self.crq_transformer_config = crq_transformer_config - self.eos_token_id = eos_token_id - self.group_size = group_size - self.enable_audio_invert_tower = enable_audio_invert_tower - self.pad_token_id = pad_token_id - - -class FunAudioChatConfig(PreTrainedConfig): - model_type = "funaudiochat" - attribute_map = { - "audio_token_id": "audio_token_index", - } - - def __init__( - self, - audio_config: PreTrainedConfig | dict | None = None, - text_config: PreTrainedConfig | dict | None = None, - audio_token_index: int = 151646, - ignore_index: int = -100, - hidden_size: int | None = None, - **kwargs, - ) -> None: - self.audio_token_index = audio_token_index - self.ignore_index = ignore_index - - if audio_config is None: - self.audio_config = FunAudioChatAudioEncoderConfig() - elif isinstance(audio_config, dict): - default_model_type = FunAudioChatAudioEncoderConfig.model_type - audio_config.setdefault("model_type", default_model_type) - self.audio_config = FunAudioChatAudioEncoderConfig(**audio_config) - else: - self.audio_config = audio_config - - if text_config is None: - self.text_config = CONFIG_MAPPING["qwen2"]() - elif isinstance(text_config, dict): - # Default to qwen2 for backwards compatibility; FunAudioChat uses - # qwen3 in practice for recent checkpoints. - text_config.setdefault("model_type", "qwen2") - self.text_config = CONFIG_MAPPING[text_config["model_type"]](**text_config) - else: - self.text_config = text_config - - self.hidden_size = ( - int(self.text_config.hidden_size) - if hidden_size is None - else int(hidden_size) - ) - - super().__init__(**kwargs) diff --git a/vllm/transformers_utils/configs/hy_v4.py b/vllm/transformers_utils/configs/hy_v4.py deleted file mode 100644 index 687a2206c8..0000000000 --- a/vllm/transformers_utils/configs/hy_v4.py +++ /dev/null @@ -1,173 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from huggingface_hub.dataclasses import strict -from transformers.configuration_utils import PreTrainedConfig -from transformers.modeling_rope_utils import RopeParameters - - -@strict -class HYV4Config(PreTrainedConfig): - r"""Configuration class for the HY V4 (`HYV4ForCausalLM`) model. - - Kept field-for-field in sync with - `transformers.models.hy_v4.configuration_hy_v4.HYV4Config` so that a - checkpoint resolves to the same values through either class. Registering it - in vLLM's `_CONFIG_REGISTRY` lets the architecture load without - `trust_remote_code` and without an `auto_map` module next to the weights. - - HYV4 is a mixture-of-experts causal language model using Multi-head Latent - Attention (MLA), DeepSeek-style sparse attention (DSA), gated MLA, - learnable attention sinks, and independent Hyper-Connections (iHC). - - The `layer_types`, `num_nextn_predict_layers` and `mtp_loss_factor` fields - below are vLLM-only additions: upstream leaves them to `**kwargs`, but - vLLM reads them directly (sparse-attention selection and MTP speculative - decoding), so they are declared to give checkpoints that omit them a - working default instead of a `None`. - - Args: - mlp_layer_types: Per-layer MLP kind, `"dense"` or `"sparse"`. Defaults - to one dense layer followed by sparse MoE layers. - layer_types: Per-layer attention kind, e.g. `"full_attention"` or - `"deepseek_sparse_attention"`. - indexer_types: Per-layer DSA indexer kind, `"full"` or `"shared"`. A - shared layer reuses the most recent full indexer in the same - forward request. - index_topk: Maximum number of key positions selected by each DSA query. - index_head_dim: Hidden dimension of each DSA indexer head. - index_n_heads: Number of DSA indexer heads. - enable_lm_head_fp32: Whether the language-model head emits float32 - logits. Surfaced as ``head_dtype`` so `LogitsProcessor` accumulates - the projection into fp32 while the weight stays in the model dtype. - enable_ihc: Whether independent Hyper-Connections are enabled. - hc_mult: Number of hidden-state channels maintained by iHC. - hc_magnitude: Scale applied to the iHC post-gating branch. - hc_eps: Numerical epsilon added to iHC sigmoid gates. - gated_mla: Whether to gate the MLA output. - gating_type: MLA gate granularity, `"elementwise"` or `"headwise"`. - learnable_sink: Whether to add a learned per-head attention sink. - learnable_sink_init: Initial value of each learned attention-sink - logit. - swiglu_limit: Magnitude of the routed-expert SwiGLU clamp. Values at or - below zero disable the clamp. - num_nextn_predict_layers: Number of MTP layers in the checkpoint. - mtp_loss_factor: Training-time MTP loss weight (unused at inference). - - """ - - model_type = "hy_v4" - keys_to_ignore_at_inference = ["past_key_values"] - attribute_map = { - "num_local_experts": "n_routed_experts", - } - base_model_tp_plan = { - "layers.*.self_attn.q_b_proj": "colwise", - "layers.*.self_attn.kv_a_proj_with_mqa": "mla_kv_a_proj", - "layers.*.self_attn.kv_b_proj": "colwise", - "layers.*.self_attn.o_proj": "rowwise", - "layers.*.self_attn.linear_gate": "colwise", - "layers.*.mlp.experts.gate_up_proj": "packed_colwise", - "layers.*.mlp.experts.down_proj": "rowwise", - "layers.*.mlp.experts": "moe_tp_experts", - "layers.*.mlp.shared_experts.gate_proj": "colwise", - "layers.*.mlp.shared_experts.up_proj": "colwise", - "layers.*.mlp.shared_experts.down_proj": "rowwise", - "layers.*.mlp.gate_proj": "colwise", - "layers.*.mlp.up_proj": "colwise", - "layers.*.mlp.down_proj": "rowwise", - } - base_model_pp_plan = { - "embed_tokens": (["input_ids"], ["inputs_embeds"]), - "layers": (["hidden_states", "attention_mask"], ["hidden_states"]), - "norm": (["hidden_states"], ["hidden_states"]), - } - - vocab_size: int = 120832 - hidden_size: int = 2816 - intermediate_size: int = 6912 - moe_intermediate_size: int = 768 - num_hidden_layers: int = 34 - num_attention_heads: int = 32 - num_key_value_heads: int = 32 - head_dim: int = 256 - hidden_act: str = "silu" - max_position_embeddings: int = 262144 - initializer_range: float = 0.006 - rms_norm_eps: float = 1e-5 - use_cache: bool = True - pad_token_id: int | None = 120002 - bos_token_id: int | None = 120000 - eos_token_id: int | list[int] | None = 120025 - tie_word_embeddings: bool = False - attention_bias: bool = False - attention_dropout: float = 0.0 - n_routed_experts: int = 256 - n_shared_experts: int = 1 - num_experts_per_tok: int = 8 - routed_scaling_factor: float = 2.827 - norm_topk_prob: bool = True - q_lora_rank: int = 1536 - kv_lora_rank: int = 512 - qk_nope_head_dim: int = 192 - qk_rope_head_dim: int = 64 - v_head_dim: int = 256 - mlp_layer_types: list[str] | None = None - layer_types: list[str] | None = None - index_topk: int = 2048 - index_head_dim: int = 128 - index_n_heads: int = 16 - indexer_types: list[str] | None = None - enable_lm_head_fp32: bool = True - enable_ihc: bool = True - hc_mult: int = 4 - hc_magnitude: float = 2.0 - hc_eps: float = 1e-6 - gated_mla: bool = True - gating_type: str = "elementwise" - learnable_sink: bool = True - learnable_sink_init: float = 0.0 - swiglu_limit: float = 10.0 - rope_parameters: RopeParameters | dict | None = None - num_nextn_predict_layers: int = 1 - mtp_loss_factor: float = 0.1 - - def __post_init__(self, **kwargs): - self.qk_head_dim = self.qk_nope_head_dim + self.qk_rope_head_dim - # RoPE applies only to the rope slice, so point `head_dim` at it: the - # inherited rotary embedding reads `config.head_dim` and then computes - # the right frequencies with no override. - self.head_dim = self.qk_rope_head_dim - - if self.mlp_layer_types is None: - self.mlp_layer_types = ["dense"] * min(1, self.num_hidden_layers) + [ - "sparse" - ] * max(self.num_hidden_layers - 1, 0) - if self.indexer_types is None: - self.indexer_types = [ - "full" if layer_idx == 0 or (layer_idx - 1) % 4 == 0 else "shared" - for layer_idx in range(self.num_hidden_layers) - ] - # `PreTrainedConfig` validates `layer_types` against a fixed allow-list - # in which "deepseek_sparse_attention" is the only accepted spelling of - # a DSA layer. The model treats "sparse_attention" and "sparse" as - # equivalent, so fold them in before the base class validates. - if self.layer_types is not None: - self.layer_types = [ - "deepseek_sparse_attention" - if lt in ("sparse_attention", "sparse") - else lt - for lt in self.layer_types - ] - - # `ModelConfig.head_dtype` reads this off the HF config, so an fp32 head - # needs no model-side dtype juggling: LogitsProcessor picks the - # `torch.mm(out_dtype=float32)` path and leaves the weight in the model - # dtype. Only fill it in when the checkpoint (or --hf-overrides) has not - # already pinned a value. - if self.enable_lm_head_fp32 and getattr(self, "head_dtype", None) is None: - self.head_dtype = "float32" - - super().__post_init__(**kwargs) - - -__all__ = ["HYV4Config"] diff --git a/vllm/transformers_utils/configs/isaac.py b/vllm/transformers_utils/configs/isaac.py deleted file mode 100644 index c46667cb2a..0000000000 --- a/vllm/transformers_utils/configs/isaac.py +++ /dev/null @@ -1,99 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from __future__ import annotations - -from transformers import Qwen3Config -from transformers.models.siglip2.configuration_siglip2 import Siglip2VisionConfig - - -class PixelShuffleSiglip2VisionConfig(Siglip2VisionConfig): - """Vision configuration for Isaac with Pixel Shuffle support. - - Extends Siglip2VisionConfig with additional fields for pixel shuffle. - """ - - model_type = "pixel_shuffle_siglip2" - base_config_key = "vision_config" - - def __init__( - self, - pixel_shuffle_scale_factor: int = 1, - num_patches: int = 256, - **kwargs, - ): - super().__init__(**kwargs) - - # Add our custom fields - self.pixel_shuffle_scale_factor = pixel_shuffle_scale_factor - self.num_patches = num_patches - - -class IsaacConfig(Qwen3Config): - """Configuration class for Isaac multimodal model.""" - - model_type = "isaac" - sub_configs = { - "vision_config": PixelShuffleSiglip2VisionConfig, - "text_config": Qwen3Config, - } - - def __init__( - self, - text_config=None, - vision_config=None, - vision_patch_size: int = 16, - vision_max_num_patches: int = 256, - vision_min_num_patches: int | None = None, - pixel_shuffle_scale: int = 1, - max_sequence_length: int = 16384, - vision_token: str = "", - vision_attn_implementation: str | None = None, - **kwargs, - ): - if isinstance(text_config, dict): - # from HF config - self.text_config = self.sub_configs["text_config"](**text_config) - elif text_config is None: - # For BC use all kwargs to init text config. - self.text_config = self.sub_configs["text_config"](**kwargs) - else: - # from Qwen3Config - self.text_config = text_config - - # EventStreamProcessor parameters (for backward compatibility) - self.video_patch_size = vision_patch_size - self.vision_max_num_patches = vision_max_num_patches - self.vision_min_num_patches = vision_min_num_patches - self.pixel_shuffle_scale = pixel_shuffle_scale - - # Processing parameters - self.max_sequence_length = max_sequence_length - self.vision_token = vision_token - - # Handle vision config - PixelShuffleSiglip2VisionConfig instance - if isinstance(vision_config, dict): - self.vision_config = PixelShuffleSiglip2VisionConfig(**vision_config) - elif vision_config is None: - self.vision_config = PixelShuffleSiglip2VisionConfig() - else: - self.vision_config = vision_config - - # Ensure compatibility with pretrained checkpoints - self.vision_config.pixel_shuffle_scale_factor = getattr( - self.vision_config, - "pixel_shuffle_scale_factor", - pixel_shuffle_scale, - ) - self.vision_config.num_patches = getattr( - self.vision_config, - "num_patches", - vision_max_num_patches, - ) - self.vision_attn_implementation = vision_attn_implementation - super().__init__(**kwargs) - - -__all__ = [ - "IsaacConfig", - "PixelShuffleSiglip2VisionConfig", -] diff --git a/vllm/transformers_utils/configs/k3_dspark.py b/vllm/transformers_utils/configs/k3_dspark.py deleted file mode 100644 index 325231627b..0000000000 --- a/vllm/transformers_utils/configs/k3_dspark.py +++ /dev/null @@ -1,77 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from transformers import DeepseekV3Config - - -class K3DSparkConfig(DeepseekV3Config): - """Configuration for a dense MLA DSpark draft model.""" - - model_type = "k3_dspark" - has_no_defaults_at_init = True - - def __init__( - self, - mla_use_nope: bool = False, - mla_use_output_gate: bool = False, - mla_use_qk_norm: bool = False, - rope_theta: float = 50000.0, - **kwargs, - ) -> None: - # DeepseekV3Config defaults to a MoE topology. Zero these fields so - # generic vLLM config logic also recognizes this draft as dense. - kwargs.setdefault("n_routed_experts", 0) - kwargs.setdefault("n_shared_experts", 0) - kwargs.setdefault("num_experts_per_tok", 0) - - rope_parameters = kwargs.get("rope_parameters") - if rope_parameters is None: - kwargs["rope_parameters"] = { - "rope_type": "default", - "rope_theta": rope_theta, - } - else: - rope_parameters = dict(rope_parameters) - rope_parameters.setdefault("rope_type", "default") - rope_parameters.setdefault("rope_theta", rope_theta) - kwargs["rope_parameters"] = rope_parameters - - super().__init__(**kwargs) - self.mla_use_nope = mla_use_nope - self.mla_use_output_gate = mla_use_output_gate - self.mla_use_qk_norm = mla_use_qk_norm - - unsupported = [ - name - for name in ( - "mla_use_nope", - "mla_use_output_gate", - "mla_use_qk_norm", - "dspark_bonus_anchor", - ) - if getattr(self, name, False) - ] - if self.q_lora_rank is None: - unsupported.append("q_lora_rank=None") - if unsupported: - raise ValueError( - "MLA DSpark does not support " + ", ".join(unsupported) + "." - ) - - self.draft_vocab_size = ( - getattr(self, "draft_vocab_size", None) or self.vocab_size - ) - if self.draft_vocab_size != self.vocab_size: - raise ValueError( - "MLA DSpark requires draft_vocab_size to equal vocab_size when " - "sharing the target embedding and LM head." - ) - - target_layer_ids = getattr(self, "target_layer_ids", None) - if not target_layer_ids or getattr(self, "num_target_layers", None) != len( - target_layer_ids - ): - raise ValueError( - "MLA DSpark requires non-empty target_layer_ids and a matching " - "num_target_layers." - ) diff --git a/vllm/transformers_utils/configs/kimi_k3.py b/vllm/transformers_utils/configs/kimi_k3.py deleted file mode 100644 index 6154a438bb..0000000000 --- a/vllm/transformers_utils/configs/kimi_k3.py +++ /dev/null @@ -1,139 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Kimi-K3 multimodal configuration.""" - -from transformers.configuration_utils import PreTrainedConfig - -from vllm.logger import init_logger -from vllm.transformers_utils.configs.kimi_linear import KimiLinearConfig - -logger = init_logger(__name__) - - -class KimiK3VisionConfig(PreTrainedConfig): - model_type = "kimi_k3_vision" - - def __init__( - self, - patch_size: int = 14, - init_pos_emb_height: int = 64, - init_pos_emb_width: int = 64, - init_pos_emb_time: int = 4, - pos_emb_type: str = "divided_fixed", - vt_num_attention_heads: int = 12, - vt_num_hidden_layers: int = 27, - vt_hidden_size: int = 1024, - vt_intermediate_size: int = 4096, - merge_kernel_size: tuple[int, int] = (2, 2), - video_attn_type: str = "spatial_temporal", - merge_type: str = "sd2_tpool", - _attn_implementation: str = "flash_attention_2", - mm_projector_type: str = "patchmergerv2", - mm_hidden_size: int | None = None, - projector_hidden_act: str = "gelu", - projector_ln_eps: float = 1e-5, - qkv_hidden_size: int = 1536, - norm_type: str = "rmsnorm", - attn_bias: bool = False, - patch_embed_proj_bias: bool = False, - mlp_type: str = "mlp2", - linear_bias: bool = False, - activation_func: str = "gelu_pytorch_tanh", - pos_emb_interpolation_mode: str = "bilinear", - text_hidden_size: int = 2304, - **kwargs, - ): - super().__init__(**kwargs) - - self.patch_size = patch_size - self.init_pos_emb_height = init_pos_emb_height - self.init_pos_emb_width = init_pos_emb_width - self.init_pos_emb_time = init_pos_emb_time - self.pos_emb_type = pos_emb_type - self.vt_num_attention_heads = vt_num_attention_heads - self.vt_num_hidden_layers = vt_num_hidden_layers - self.vt_hidden_size = vt_hidden_size - self.vt_intermediate_size = vt_intermediate_size - self.merge_kernel_size = tuple(merge_kernel_size) - self.video_attn_type = video_attn_type - self.merge_type = merge_type - self._attn_implementation = _attn_implementation - - self.mm_projector_type = mm_projector_type - self.mm_hidden_size = ( - mm_hidden_size if mm_hidden_size is not None else vt_hidden_size - ) - self.projector_hidden_act = projector_hidden_act - self.projector_ln_eps = projector_ln_eps - self.text_hidden_size = text_hidden_size - - self.qkv_hidden_size = qkv_hidden_size - self.norm_type = norm_type - self.attn_bias = attn_bias - self.patch_embed_proj_bias = patch_embed_proj_bias - self.mlp_type = mlp_type - self.linear_bias = linear_bias - self.activation_func = activation_func - self.pos_emb_interpolation_mode = pos_emb_interpolation_mode - - # Aliases consumed by the Kimi-K2.5 vision implementation. - self.num_attention_heads = vt_num_attention_heads - self.num_hidden_layers = vt_num_hidden_layers - self.hidden_size = vt_hidden_size - self.intermediate_size = vt_intermediate_size - - -class KimiK3Config(PreTrainedConfig): - model_type = "kimi_k3" - - def __init__( - self, - text_config: dict | KimiLinearConfig | None = None, - vision_config: dict | KimiK3VisionConfig | None = None, - ignore_index: int = -100, - media_placeholder_token_id: int = 163605, - pad_token_id: int = 0, - image_placeholder: str = "<|kimi_image_placeholder|>", - **kwargs, - ): - if text_config is None: - self.text_config = KimiLinearConfig() - elif isinstance(text_config, dict): - self.text_config = KimiLinearConfig(**text_config) - else: - self.text_config = text_config - - if vision_config is None: - self.vision_config = KimiK3VisionConfig() - elif isinstance(vision_config, dict): - self.vision_config = KimiK3VisionConfig(**vision_config) - else: - self.vision_config = vision_config - - # K3's vision projector output must match the text model's hidden size. - # Override any value provided in vision_config for safety. - if self.vision_config.text_hidden_size != self.text_config.hidden_size: - logger.info( - "Overriding vision_config.text_hidden_size from %s to %s " - "to match text_config.hidden_size", - self.vision_config.text_hidden_size, - self.text_config.hidden_size, - ) - self.vision_config.text_hidden_size = self.text_config.hidden_size - - self.ignore_index = ignore_index - self.media_placeholder_token_id = media_placeholder_token_id - self.image_placeholder = image_placeholder - - if getattr(self.text_config, "quantization_config", None) is not None: - self.quantization_config = self.text_config.quantization_config - - super().__init__(pad_token_id=pad_token_id, **kwargs) - - @property - def hidden_size(self) -> int: - return self.text_config.hidden_size - - @property - def vocab_size(self) -> int: - return self.text_config.vocab_size diff --git a/vllm/transformers_utils/configs/kimi_linear.py b/vllm/transformers_utils/configs/kimi_linear.py deleted file mode 100644 index 62b3963e1c..0000000000 --- a/vllm/transformers_utils/configs/kimi_linear.py +++ /dev/null @@ -1,161 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from transformers.configuration_utils import PreTrainedConfig - - -class KimiLinearConfig(PreTrainedConfig): - model_type = "kimi_linear" - keys_to_ignore_at_inference = ["past_key_values"] - - def __init__( - self, - model_type="kimi_linear", - vocab_size=163840, - hidden_size=4096, - head_dim=None, - intermediate_size=11008, - num_hidden_layers=32, - num_attention_heads=32, - num_key_value_heads=None, - hidden_act="silu", - initializer_range=0.02, - rms_norm_eps=1e-6, - use_cache=True, - pad_token_id=0, - bos_token_id=1, - eos_token_id=2, - rope_parameters=None, - tie_word_embeddings=False, - moe_intermediate_size: int | None = None, - moe_renormalize: bool = True, - moe_router_activation_func: str = "sigmoid", - num_experts: int | None = None, - num_experts_per_token: int | None = None, - num_shared_experts: int = 0, - routed_scaling_factor: float = 1.0, - first_k_dense_replace: int = 0, - moe_layer_freq: int = 1, - use_grouped_topk: bool = True, - num_expert_group: int = 1, - topk_group: int = 1, - q_lora_rank: int | None = None, - kv_lora_rank: int | None = None, - qk_nope_head_dim: int | None = None, - qk_rope_head_dim: int | None = None, - v_head_dim: int | None = None, - mla_use_nope: bool | None = False, - mla_use_output_gate: bool | None = False, - num_nextn_predict_layers: int = 0, - linear_attn_config: dict | None = None, - attn_res_block_size: int | None = None, - latent_moe_use_norm: bool = False, - activation_situ_beta: float | None = None, - activation_situ_linear_beta: float | None = None, - max_position_embeddings: int = 4096, - routed_expert_hidden_size: int | None = None, - topk_method: str = "noaux_tc", - **kwargs, - ): - self.model_type = model_type - self.vocab_size = vocab_size - self.hidden_size = hidden_size - self.head_dim = ( - head_dim if head_dim is not None else hidden_size // num_attention_heads - ) - self.intermediate_size = intermediate_size - self.num_hidden_layers = num_hidden_layers - self.num_attention_heads = num_attention_heads - - # for backward compatibility - if num_key_value_heads is None: - num_key_value_heads = num_attention_heads - - self.num_key_value_heads = num_key_value_heads - self.hidden_act = hidden_act - self.initializer_range = initializer_range - self.rms_norm_eps = rms_norm_eps - self.use_cache = use_cache - # Try to set `rope_scaling` if available, otherwise use `rope_parameters` - rope_scaling = kwargs.pop("rope_scaling", None) - rope_parameters = rope_scaling or rope_parameters or {"rope_type": "default"} - rope_theta = kwargs.pop("rope_theta", 10000.0) - if "rope_theta" not in rope_parameters: - rope_parameters["rope_theta"] = rope_theta - self.rope_parameters = rope_parameters - - self.q_lora_rank = q_lora_rank - self.kv_lora_rank = kv_lora_rank - self.qk_nope_head_dim = qk_nope_head_dim - self.qk_rope_head_dim = qk_rope_head_dim - self.v_head_dim = v_head_dim - self.mla_use_nope = mla_use_nope - self.mla_use_output_gate = mla_use_output_gate - # moe config - self.num_experts = num_experts - self.num_experts_per_token = num_experts_per_token - self.moe_renormalize = moe_renormalize - self.num_shared_experts = num_shared_experts - self.routed_scaling_factor = routed_scaling_factor - self.moe_router_activation_func = moe_router_activation_func - assert self.moe_router_activation_func in ("softmax", "sigmoid") - self.moe_intermediate_size = moe_intermediate_size - self.first_k_dense_replace = first_k_dense_replace - self.moe_layer_freq = moe_layer_freq - self.use_grouped_topk = use_grouped_topk - self.num_expert_group = num_expert_group - self.topk_group = topk_group - self.num_nextn_predict_layers = num_nextn_predict_layers - - self.attn_res_block_size = attn_res_block_size - self.latent_moe_use_norm = latent_moe_use_norm - self.activation_situ_beta = activation_situ_beta - self.activation_situ_linear_beta = activation_situ_linear_beta - self.max_position_embeddings = max_position_embeddings - self.routed_expert_hidden_size = routed_expert_hidden_size - self.topk_method = topk_method - - if linear_attn_config is not None: - assert linear_attn_config["kda_layers"] is not None - assert linear_attn_config["full_attn_layers"] is not None - self.linear_attn_config = linear_attn_config - - super().__init__( - pad_token_id=pad_token_id, - bos_token_id=bos_token_id, - eos_token_id=eos_token_id, - tie_word_embeddings=tie_word_embeddings, - **kwargs, - ) - - @property - def is_mla(self): - return ( - self.q_lora_rank is not None - or self.kv_lora_rank is not None - or self.qk_nope_head_dim is not None - or self.qk_rope_head_dim is not None - or self.v_head_dim is not None - or self.mla_use_nope is True - ) - - @property - def is_moe(self): - return self.num_experts is not None - - @property - def is_linear_attn(self) -> bool: - return not ( - self.linear_attn_config is None - or ( - isinstance(self.linear_attn_config, dict) - and self.linear_attn_config["kda_layers"] is not None - and len(self.linear_attn_config["kda_layers"]) == 0 - ) - ) - - def is_kda_layer(self, layer_idx: int): - return ( - self.linear_attn_config is not None - and (layer_idx + 1) in self.linear_attn_config["kda_layers"] - ) diff --git a/vllm/transformers_utils/configs/kimi_vl.py b/vllm/transformers_utils/configs/kimi_vl.py deleted file mode 100644 index 1b2d32a263..0000000000 --- a/vllm/transformers_utils/configs/kimi_vl.py +++ /dev/null @@ -1,38 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -# Adapted from https://huggingface.co/moonshotai/Kimi-VL-A3B-Instruct/blob/main/configuration_kimi_vl.py - -from transformers import DeepseekV2Config -from transformers.configuration_utils import PreTrainedConfig - -from vllm.transformers_utils.configs.moonvit import MoonViTConfig - - -class KimiVLConfig(PreTrainedConfig): - model_type = "kimi_vl" - - def __init__( - self, - vision_config: dict | MoonViTConfig | None = None, - text_config: dict | DeepseekV2Config | None = None, - ignore_index: int = -100, - media_placeholder_token_id: int = 163605, - pad_token_id: int = 0, - **kwargs, - ): - if vision_config is None: - vision_config = MoonViTConfig() - elif isinstance(vision_config, dict): - vision_config = MoonViTConfig(**vision_config) - self.vision_config = vision_config - - if text_config is None: - text_config = DeepseekV2Config() - elif isinstance(text_config, dict): - text_config = DeepseekV2Config(**text_config) - self.text_config: DeepseekV2Config = text_config - - self.ignore_index = ignore_index - self.media_placeholder_token_id = media_placeholder_token_id - - super().__init__(pad_token_id=pad_token_id, **kwargs) diff --git a/vllm/transformers_utils/configs/midashenglm.py b/vllm/transformers_utils/configs/midashenglm.py deleted file mode 100644 index 76d8393e01..0000000000 --- a/vllm/transformers_utils/configs/midashenglm.py +++ /dev/null @@ -1,103 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -# Copyright 2025 Horizon team, Xiaomi MiLM Plus. -# Copyright 2024 The Qwen team. -# Copyright 2023 The vLLM team. -# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved. -# -# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX -# and OPT implementations in this library. It has been modified from its -# original forms to accommodate minor architectural differences compared -# to GPT-NeoX and OPT used by the Meta AI team that trained the model. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from transformers import PreTrainedConfig -from transformers.models.qwen2_5_omni.configuration_qwen2_5_omni import ( - Qwen2_5OmniTextConfig, -) - - -class DashengConfig(PreTrainedConfig): - model_type = "midashenglm_dasheng_encoder" - - def __init__( - self, - embed_dim: int = 768, - outputdim: int = 527, - patch_size: int | tuple[int, int] = 16, - patch_stride: int | tuple[int, int] = 16, - input_channels: int = 1, - target_length: int = 1012, - depth: int = 12, - num_heads: int = 12, - mlp_ratio: float = 4.0, - qkv_bias: bool = True, - init_values: float | None = None, - drop_rate: float = 0.0, - attn_drop_rate: float = 0.0, - f_min: float = 0.0, - f_max: float = 8000.0, - center: bool = True, - win_length: int = 512, - hop_length: int = 160, - sample_rate: int = 16000, - n_fft: int = 512, - n_mels: int = 64, - **kwargs, - ): - self.embed_dim = embed_dim - self.outputdim = outputdim - self.patch_size = patch_size - self.patch_stride = patch_stride - self.input_channels = input_channels - self.target_length = target_length - self.depth = depth - self.num_heads = num_heads - self.mlp_ratio = mlp_ratio - self.qkv_bias = qkv_bias - self.init_values = init_values - self.drop_rate = drop_rate - self.attn_drop_rate = attn_drop_rate - self.f_min = f_min - self.f_max = f_max - self.center = center - self.win_length = win_length - self.hop_length = hop_length - self.sample_rate = sample_rate - self.n_fft = n_fft - self.n_mels = n_mels - super().__init__(**kwargs) - - -class MiDashengLMConfig(PreTrainedConfig): - model_type = "midashenglm" - - def __init__( - self, - audio_encoder_config: dict | None = None, - subsample_factor: int = 5, - text_config: dict | None = None, - audio_token_id: int | None = None, - **kwargs, - ): - self.audio_encoder_config = DashengConfig(**(audio_encoder_config or {})) - self.subsample_factor = subsample_factor - self.text_config = ( - Qwen2_5OmniTextConfig(**text_config) - if text_config - else Qwen2_5OmniTextConfig() - ) - self.text_config.rope_parameters = None # uses_mrope is false - self.audio_token_id = audio_token_id - super().__init__(**kwargs) diff --git a/vllm/transformers_utils/configs/minimax_m3.py b/vllm/transformers_utils/configs/minimax_m3.py deleted file mode 100644 index 374b854a66..0000000000 --- a/vllm/transformers_utils/configs/minimax_m3.py +++ /dev/null @@ -1,149 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from typing import Any - -from transformers import PreTrainedConfig - - -class MiniMaxM3TextConfig(PreTrainedConfig): - """Config for the MiniMax M3 text backbone (MiniMaxM3SparseForCausalLM). - - Defaults mirror the ``text_config`` of the MiniMax-M3-preview checkpoint. - """ - - model_type = "minimax_m3_text" - architectures = ["MiniMaxM3SparseForCausalLM"] - - def __init__( - self, - vocab_size: int = 200064, - hidden_size: int = 6144, - intermediate_size: int = 3072, - dense_intermediate_size: int = 12288, - shared_intermediate_size: int = 3072, - num_hidden_layers: int = 60, - num_attention_heads: int = 64, - num_key_value_heads: int = 4, - head_dim: int = 128, - max_position_embeddings: int = 524288, - rms_norm_eps: float = 1e-6, - use_gemma_norm: bool = True, - attention_output_gate: bool = False, - rope_theta: float = 5000000, - rotary_dim: int = 64, - partial_rotary_factor: float = 0.5, - hidden_act: str = "swigluoai", - swiglu_alpha: float = 1.702, - # SwiGLU-OAI uses the (up + 1) bias, i.e. beta=1.0 (matches the - # reference: gate * sigmoid(gate * alpha) * (up + 1)). The checkpoint - # config omits swiglu_beta, so this default must stay 1.0. - swiglu_beta: float = 1.0, - swiglu_limit: float = 7.0, - use_qk_norm: bool = True, - qk_norm_type: str = "per_head", - num_local_experts: int = 128, - num_experts_per_tok: int = 4, - n_shared_experts: int = 1, - scoring_func: str = "sigmoid", - use_routing_bias: bool = True, - routed_scaling_factor: float = 2.0, - num_mtp_modules: int = 1, - moe_layer_freq: list[int] | None = None, - sparse_attention_config: dict[str, Any] | None = None, - tie_word_embeddings: bool = False, - **kwargs, - ): - self.vocab_size = vocab_size - self.hidden_size = hidden_size - self.intermediate_size = intermediate_size - self.dense_intermediate_size = dense_intermediate_size - self.shared_intermediate_size = shared_intermediate_size - self.num_hidden_layers = num_hidden_layers - self.num_attention_heads = num_attention_heads - self.num_key_value_heads = num_key_value_heads - self.head_dim = head_dim - self.max_position_embeddings = max_position_embeddings - self.rms_norm_eps = rms_norm_eps - self.use_gemma_norm = use_gemma_norm - self.attention_output_gate = attention_output_gate - self.rope_theta = rope_theta - self.rotary_dim = rotary_dim - self.partial_rotary_factor = partial_rotary_factor - self.hidden_act = hidden_act - self.swiglu_alpha = swiglu_alpha - self.swiglu_beta = swiglu_beta - self.swiglu_limit = swiglu_limit - self.use_qk_norm = use_qk_norm - self.qk_norm_type = qk_norm_type - self.num_local_experts = num_local_experts - self.num_experts_per_tok = num_experts_per_tok - self.n_shared_experts = n_shared_experts - self.scoring_func = scoring_func - self.use_routing_bias = use_routing_bias - self.routed_scaling_factor = routed_scaling_factor - self.num_mtp_modules = num_mtp_modules - # First 3 layers are dense; the remaining 57 are sparse MoE. - self.moe_layer_freq = ( - moe_layer_freq if moe_layer_freq is not None else [0] * 3 + [1] * 57 - ) - self.sparse_attention_config = ( - sparse_attention_config - if sparse_attention_config is not None - else { - "use_sparse_attention": True, - "sparse_index_dim": 128, - "sparse_num_index_heads": 4, - "sparse_topk_blocks": 16, - "sparse_block_size": 128, - "sparse_disable_index_value": [0] * 3 + [1] * 57, - "sparse_score_type": "max", - "sparse_init_block": 0, - "sparse_local_block": 1, - "sparse_attention_freq": [0] * 3 + [1] * 57, - } - ) - super().__init__(tie_word_embeddings=tie_word_embeddings, **kwargs) - - -class MiniMaxM3MTPConfig(MiniMaxM3TextConfig): - """Config for a standalone MiniMax M3 MTP (multi-token prediction) head. - - The MTP transformer layer is structurally a single MiniMax M3 decoder - layer, so this reuses the text backbone schema. Standalone MTP checkpoints - use ``model_type='minimax_m3_mtp'`` and a single hidden layer. - """ - - model_type = "minimax_m3_mtp" - architectures = ["MiniMaxM3MTP"] - - def __init__(self, num_hidden_layers: int = 1, **kwargs): - super().__init__(num_hidden_layers=num_hidden_layers, **kwargs) - - -class MiniMaxM3Config(PreTrainedConfig): - """Top-level MiniMax M3 (VL) config. - - Holds the text backbone as ``text_config`` so that - ``config.get_text_config()`` extracts the MiniMaxM3SparseForCausalLM - backbone. Vision components are kept as a raw dict passthrough and are - not modeled here. - """ - - model_type = "minimax_m3_vl" - - def __init__( - self, - text_config: dict | MiniMaxM3TextConfig | None = None, - vision_config: dict | None = None, - **kwargs, - ): - if text_config is None: - text_config = MiniMaxM3TextConfig() - elif isinstance(text_config, dict): - text_config = MiniMaxM3TextConfig(**text_config) - self.text_config = text_config - self.vision_config = vision_config - - self.hidden_size = text_config.hidden_size - - super().__init__(**kwargs) diff --git a/vllm/transformers_utils/configs/moondream3.py b/vllm/transformers_utils/configs/moondream3.py deleted file mode 100644 index e8a80082c1..0000000000 --- a/vllm/transformers_utils/configs/moondream3.py +++ /dev/null @@ -1,153 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Configuration for Moondream3 model.""" - -from transformers import PreTrainedConfig - - -class Moondream3VisionConfig(PreTrainedConfig): - """Vision encoder configuration for Moondream3.""" - - model_type = "moondream3_vision" - - def __init__( - self, - enc_dim: int = 1152, - enc_patch_size: int = 14, - enc_n_layers: int = 27, - enc_ff_dim: int = 4304, - enc_n_heads: int = 16, - proj_inner_dim: int = 8192, - crop_size: int = 378, - max_crops: int = 12, - overlap_margin: int = 4, - **kwargs, - ): - super().__init__(**kwargs) - self.enc_dim = enc_dim - self.enc_patch_size = enc_patch_size - self.enc_n_layers = enc_n_layers - self.enc_ff_dim = enc_ff_dim - self.enc_n_heads = enc_n_heads - self.proj_inner_dim = proj_inner_dim - self.crop_size = crop_size - self.max_crops = max_crops - self.overlap_margin = overlap_margin - - # Standard HuggingFace attributes for vision config - self.hidden_size = enc_dim - self.num_attention_heads = enc_n_heads - self.num_hidden_layers = enc_n_layers - self.intermediate_size = enc_ff_dim - self.patch_size = enc_patch_size - self.image_size = crop_size - - -class Moondream3TextConfig(PreTrainedConfig): - """Text decoder configuration for Moondream3.""" - - model_type = "moondream3_text" - - def __init__( - self, - dim: int = 2048, - ff_dim: int = 8192, - n_layers: int = 24, - vocab_size: int = 51200, - max_context: int = 4096, - n_heads: int = 32, - n_kv_heads: int = 32, - prefix_attn: int = 730, - rope_theta: float = 1500000.0, - moe: dict | None = None, - **kwargs, - ): - super().__init__(**kwargs) - - # Store original moondream3 config names - self.dim = dim - self.ff_dim = ff_dim - self.n_layers = n_layers - self.n_heads = n_heads - self.n_kv_heads = n_kv_heads - self.prefix_attn = prefix_attn - self.max_context = max_context - self.rope_theta = rope_theta - - # MoE config - moe = moe or {} - self.moe_start_layer = moe.get("start_layer", 4) - self.moe_num_experts = moe.get("n_experts", 64) - self.moe_experts_per_token = moe.get("n_experts_per_tok", 8) - self.moe_expert_inner_dim = moe.get("expert_inner_dim", 1024) - - # Standard HuggingFace attributes (required by vLLM) - self.hidden_size = dim - self.num_attention_heads = n_heads - self.num_key_value_heads = n_kv_heads - self.num_hidden_layers = n_layers - self.intermediate_size = ff_dim - self.vocab_size = vocab_size - self.max_position_embeddings = max_context - - # Moondream3 uses token 0 (<|endoftext|>) as both BOS and EOS. - # Token 3 (<|md_reserved_2|>) is an answer delimiter that the model - # implementation suppresses during generation. - self.bos_token_id = 0 - self.eos_token_id = 0 - - # MoE standard attributes - self.num_local_experts = self.moe_num_experts - self.num_experts_per_tok = self.moe_experts_per_token - - -class Moondream3Config(PreTrainedConfig): - """Combined configuration for Moondream3 multimodal model.""" - - model_type = "moondream3" - is_composition = True - - def __init__( - self, - config: dict | None = None, - **kwargs, - ): - config = config or {} - - # Parse text config - text_config = config.get("text", {}) - self.text_config: Moondream3TextConfig = Moondream3TextConfig(**text_config) - - # Parse vision config - vision_config = config.get("vision", {}) - self.vision_config = Moondream3VisionConfig(**vision_config) - - # Store the original config dict for model access - self.config = config - tokenizer_config = config.get("tokenizer", {}) - self.answer_token_id = tokenizer_config.get("answer_id", 3) - - super().__init__(**kwargs) - - # Expose key attributes at top level for vLLM compatibility - self.hidden_size = self.text_config.hidden_size - self.num_attention_heads = self.text_config.num_attention_heads - self.num_key_value_heads = self.text_config.num_key_value_heads - self.num_hidden_layers = self.text_config.num_hidden_layers - self.vocab_size = self.text_config.vocab_size - self.intermediate_size = self.text_config.intermediate_size - - # Moondream3 uses token 0 (<|endoftext|>) as both BOS and EOS. - # Token 3 (<|md_reserved_2|>) is an answer delimiter that the model - # implementation suppresses during generation. - self.bos_token_id = 0 - self.eos_token_id = 0 - - def get_text_config(self, decoder: bool = False) -> "Moondream3TextConfig": - """Return the text config for vLLM's text_config detection. - - Args: - decoder: Ignored. Only used for encoder-decoder models. - - """ - return self.text_config diff --git a/vllm/transformers_utils/configs/moonvit.py b/vllm/transformers_utils/configs/moonvit.py deleted file mode 100644 index c7cd34d41d..0000000000 --- a/vllm/transformers_utils/configs/moonvit.py +++ /dev/null @@ -1,33 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -# Adapted from https://huggingface.co/moonshotai/Kimi-VL-A3B-Instruct/blob/main/configuration_kimi_vl.py -from transformers.configuration_utils import PreTrainedConfig - - -class MoonViTConfig(PreTrainedConfig): - model_type = "moonvit" - - def __init__( - self, - patch_size: int = 14, - init_pos_emb_height: int = 64, - init_pos_emb_width: int = 64, - num_attention_heads: int = 16, - num_hidden_layers: int = 27, - hidden_size: int = 1152, - intermediate_size: int = 4304, - merge_kernel_size: tuple[int, int] = (2, 2), - **kwargs, - ): - super().__init__(**kwargs) - self.patch_size = patch_size - # Positional embedding config - self.init_pos_emb_height = init_pos_emb_height - self.init_pos_emb_width = init_pos_emb_width - # Transformer config - self.num_hidden_layers = num_hidden_layers - self.num_attention_heads = num_attention_heads - self.hidden_size = hidden_size - self.intermediate_size = intermediate_size - # Patch merger config - self.merge_kernel_size = merge_kernel_size diff --git a/vllm/transformers_utils/configs/moss_transcribe_diarize.py b/vllm/transformers_utils/configs/moss_transcribe_diarize.py deleted file mode 100644 index 6d57642f39..0000000000 --- a/vllm/transformers_utils/configs/moss_transcribe_diarize.py +++ /dev/null @@ -1,108 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from typing import Any - -from transformers import PreTrainedConfig, Qwen3Config -from transformers.models.whisper.configuration_whisper import WhisperConfig - - -class MossTranscribeDiarizeConfig(PreTrainedConfig): - """Configuration for MOSS-Transcribe-Diarize.""" - - model_type = "moss_transcribe_diarize" - sub_configs = {"text_config": Qwen3Config, "audio_config": WhisperConfig} - keys_to_ignore_at_inference = ["past_key_values"] - - def __init__( - self, - text_config: dict[str, Any] | Qwen3Config | None = None, - audio_config: dict[str, Any] | WhisperConfig | None = None, - audio_token_id: int = 151671, - audio_merge_size: int = 4, - adaptor_input_dim: int | None = None, - tie_word_embeddings: bool = True, - **kwargs: Any, - ) -> None: - text_config_obj: Qwen3Config - if text_config is None: - text_config_obj = Qwen3Config( - vocab_size=151936, - hidden_size=1024, - intermediate_size=3072, - num_hidden_layers=28, - num_attention_heads=16, - num_key_value_heads=8, - head_dim=128, - max_position_embeddings=40960, - tie_word_embeddings=tie_word_embeddings, - rope_theta=1_000_000.0, - layer_types=["full_attention"] * 28, - ) - elif isinstance(text_config, dict): - text_config_obj = Qwen3Config(**text_config) - else: - text_config_obj = text_config - - audio_config_obj: WhisperConfig - if audio_config is None: - audio_config_obj = WhisperConfig( - num_mel_bins=80, - d_model=1024, - encoder_layers=24, - encoder_attention_heads=16, - encoder_ffn_dim=4096, - max_source_positions=1500, - dropout=0.0, - attention_dropout=0.0, - activation_dropout=0.0, - activation_function="gelu", - encoder_layerdrop=0.0, - scale_embedding=False, - ) - elif isinstance(audio_config, dict): - audio_config_obj = WhisperConfig(**audio_config) - else: - audio_config_obj = audio_config - - text_config_obj.tie_word_embeddings = tie_word_embeddings - if not getattr(text_config_obj, "layer_types", None): - text_config_obj.layer_types = [ - "full_attention" - ] * text_config_obj.num_hidden_layers - - super().__init__(tie_word_embeddings=tie_word_embeddings, **kwargs) - - self.text_config = text_config_obj - self.audio_config = audio_config_obj - self.audio_token_id = int(audio_token_id) - self.audio_merge_size = int(audio_merge_size) - self.adaptor_input_dim = ( - int(adaptor_input_dim) - if adaptor_input_dim is not None - else int(audio_config_obj.d_model) * int(audio_merge_size) - ) - - self.vocab_size = int(text_config_obj.vocab_size) - self.hidden_size = int(text_config_obj.hidden_size) - self.intermediate_size = int(text_config_obj.intermediate_size) - self.num_hidden_layers = int(text_config_obj.num_hidden_layers) - self.num_attention_heads = int(text_config_obj.num_attention_heads) - self.num_key_value_heads = int(text_config_obj.num_key_value_heads) - self.head_dim = int(text_config_obj.head_dim) - self.hidden_act = text_config_obj.hidden_act - self.max_position_embeddings = int(text_config_obj.max_position_embeddings) - self.rms_norm_eps = float(text_config_obj.rms_norm_eps) - rope_parameters = getattr(text_config_obj, "rope_parameters", None) - rope_theta = float(getattr(text_config_obj, "rope_theta", 1_000_000.0)) - if rope_parameters is None: - rope_parameters = { - "rope_type": "default", - "rope_theta": rope_theta, - } - text_config_obj.rope_parameters = rope_parameters - self.rope_parameters = rope_parameters - self.rope_theta = float(rope_parameters.get("rope_theta", rope_theta)) - self.attention_bias = bool(text_config_obj.attention_bias) - self.attention_dropout = float(text_config_obj.attention_dropout) - self.is_causal = True diff --git a/vllm/transformers_utils/configs/muse_glimmer.py b/vllm/transformers_utils/configs/muse_glimmer.py deleted file mode 100644 index 80c4c4e171..0000000000 --- a/vllm/transformers_utils/configs/muse_glimmer.py +++ /dev/null @@ -1,389 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""MuseGlimmer model configuration for vLLM. - -Native vLLM copy of the MuseGlimmer HuggingFace configs -(``configuration_muse_glimmer.py``). MuseGlimmer's -``model_type`` is not yet registered in released transformers, so vLLM ships -this config so partners can serve MuseGlimmer checkpoints *without* trust_remote_code. - -The field set and defaults are kept byte-for-byte in sync with the HF reference -so a checkpoint's ``config.json`` deserializes identically here. Both the text -and vision configs are consumed by the native multimodal serving path. -""" - -from __future__ import annotations - -from transformers import Qwen3Config -from transformers.configuration_utils import PreTrainedConfig - - -def _default_no_rope_layers(num_hidden_layers: int) -> list[int]: - # iRoPE mask: NoPE every 4 layers, counted backward from the last layer. - stride = 4 - return [ - 0 if (num_hidden_layers - 1 - i) % stride == 0 else 1 - for i in range(num_hidden_layers) - ] - - -class MuseGlimmerTextConfig(PreTrainedConfig): - model_type = "muse_glimmer_text" - keys_to_ignore_at_inference = ["past_key_values"] - - def __init__( - self, - vocab_size: int = 202_048, - hidden_size: int = 6656, - intermediate_size: int = 19968, - num_hidden_layers: int = 52, - num_attention_heads: int = 32, - num_key_value_heads: int = 2, - head_dim: int = 128, - hidden_activation: str = "silu", - max_position_embeddings: int = 16_384, - initializer_range: float = 0.02, - rms_norm_eps: float = 1e-5, - use_cache: bool = True, - pad_token_id: int | None = None, - eos_token_id: int | list[int] | None = 200_001, - bos_token_id: int | None = 200_000, - tie_word_embeddings: bool = False, - rope_parameters: dict | None = None, - rope_theta: float | None = None, - attention_bias: bool = False, - attention_dropout: float = 0.0, - query_pre_attn_scalar: int = 256, - sliding_window: int | None = 2048, - layer_types: list[str] | None = None, - final_logit_softcapping: float | None = 20.0, - attn_logit_softcapping: float | None = None, - use_bidirectional_attention: bool | None = None, - # MuseGlimmer-specific - qk_scale_factor: float = 43.7840518911, - use_qk_norm: bool = True, - use_attn_output_gate: bool = True, - output_multiplier: float = 0.19611613513818404, - normalize_tok_embeddings: bool = True, - post_norm_eps: float = 1e-8, - no_rope_layers: list[int] | None = None, - **kwargs, - ) -> None: - self.vocab_size = vocab_size - self.hidden_size = hidden_size - self.intermediate_size = intermediate_size - self.num_hidden_layers = num_hidden_layers - self.num_attention_heads = num_attention_heads - self.num_key_value_heads = num_key_value_heads - self.head_dim = head_dim - self.hidden_activation = hidden_activation - self.max_position_embeddings = max_position_embeddings - self.initializer_range = initializer_range - self.rms_norm_eps = rms_norm_eps - self.use_cache = use_cache - self.attention_bias = attention_bias - self.attention_dropout = attention_dropout - self.query_pre_attn_scalar = query_pre_attn_scalar - self.sliding_window = sliding_window - self.final_logit_softcapping = final_logit_softcapping - self.attn_logit_softcapping = attn_logit_softcapping - self.use_bidirectional_attention = use_bidirectional_attention - - # RoPE: accept either an explicit rope_parameters dict (HF 5.x) or a - # bare rope_theta; normalize to rope_parameters for vLLM's get_rope. - if rope_parameters is None: - theta = rope_theta if rope_theta is not None else 500_000.0 - rope_parameters = {"rope_type": "default", "rope_theta": theta} - self.rope_parameters = rope_parameters - # vLLM reads rope_theta off the config in some codepaths. - self.rope_theta = rope_parameters.get("rope_theta", 500_000.0) - - # MuseGlimmer-specific fields - self.qk_scale_factor = qk_scale_factor - self.use_qk_norm = use_qk_norm - self.use_attn_output_gate = use_attn_output_gate - self.output_multiplier = output_multiplier - self.normalize_tok_embeddings = normalize_tok_embeddings - self.post_norm_eps = post_norm_eps - - self.no_rope_layers = ( - no_rope_layers - if no_rope_layers is not None - else _default_no_rope_layers(num_hidden_layers) - ) - if layer_types is None: - layer_types = [ - "full_attention" if self.no_rope_layers[i] == 0 else "sliding_attention" - for i in range(num_hidden_layers) - ] - self.layer_types = layer_types - - super().__init__( - pad_token_id=pad_token_id, - bos_token_id=bos_token_id, - eos_token_id=eos_token_id, - tie_word_embeddings=tie_word_embeddings, - **kwargs, - ) - - -class MuseGlimmerVisionConfig(PreTrainedConfig): - model_type = "muse_glimmer_vision" - - def __init__( - self, - patch_size: int = 14, - pos_emb_height: int = 32, - pos_emb_width: int = 32, - num_attention_heads: int = 16, - num_hidden_layers: int = 50, - hidden_size: int = 1536, - intermediate_size: int = 8960, - hidden_act: str = "gelu", - merge_kernel_size: int = 2, - rope_parameters: dict | None = None, - max_position_embeddings: int = 32 * 32, - output_dim: int = 6144, - patch_temporal: int = 2, - adapter_dim: int = 4096, - layer_norm_eps: float = 1e-5, - layer_types: list[str] | None = None, - **kwargs, - ) -> None: - super().__init__(**kwargs) - self.patch_size = patch_size - self.pos_emb_height = pos_emb_height - self.pos_emb_width = pos_emb_width - self.num_attention_heads = num_attention_heads - self.num_hidden_layers = num_hidden_layers - self.hidden_size = hidden_size - self.intermediate_size = intermediate_size - self.hidden_act = hidden_act - self.merge_kernel_size = merge_kernel_size - self.rope_parameters = rope_parameters - self.max_position_embeddings = max_position_embeddings - self.output_dim = output_dim - self.patch_temporal = patch_temporal - self.adapter_dim = adapter_dim - self.layer_norm_eps = layer_norm_eps - if layer_types is None: - stride = 4 - layer_types = [ - "full_attention" - if (i + 1) % stride == 0 or i == num_hidden_layers - 1 - else "sliding_attention" - for i in range(num_hidden_layers) - ] - self.layer_types = layer_types - - -class MuseGlimmerConfig(PreTrainedConfig): - model_type = "muse_glimmer" - sub_configs = { - "text_config": MuseGlimmerTextConfig, - "vision_config": MuseGlimmerVisionConfig, - } - - # --- Flat (legacy-converter) -> canonical normalization ------------------- - # MuseGlimmer checkpoints exist in two config layouts in the wild: - # - # * CANONICAL (current HF converter, transformers 5.15): nested - # ``text_config`` / ``vision_config`` sub-dicts with canonical field - # names (``hidden_activation``, ``final_logit_softcapping``, - # ``rope_parameters``, ``vision_config.hidden_size`` ...). - # - # * FLAT (older converter, e.g. Ruan's ``rl_v1/hf``, transformers 5.9): - # every field is a top-level key, with different names - # (``hidden_act``, ``output_soft_cap_temp``, ``rope_theta``, - # ``vision_latent_dim`` ...) and NO ``text_config`` nesting. - # - # Without normalization a flat config silently deserializes to an - # ALL-DEFAULT text config (every checkpoint value ignored) — a dangerous - # correctness bug: a checkpoint whose arch differs from the defaults would - # load into a wrong-shaped model with no error. So when we detect a flat - # config we hoist the flat fields into ``text_config`` / ``vision_config`` - # and rename them to canonical names. - - # flat text field name -> canonical MuseGlimmerTextConfig field name - _FLAT_TEXT_RENAMES = { - "hidden_act": "hidden_activation", - "output_soft_cap_temp": "final_logit_softcapping", - } - # canonical MuseGlimmerTextConfig constructor params that may appear flat - _FLAT_TEXT_KEYS = frozenset( - { - "vocab_size", - "hidden_size", - "intermediate_size", - "num_hidden_layers", - "num_attention_heads", - "num_key_value_heads", - "head_dim", - "hidden_activation", - "hidden_act", - "max_position_embeddings", - "initializer_range", - "rms_norm_eps", - "use_cache", - "tie_word_embeddings", - "rope_parameters", - "rope_theta", - "attention_bias", - "attention_dropout", - "query_pre_attn_scalar", - "sliding_window", - "layer_types", - "final_logit_softcapping", - "output_soft_cap_temp", - "attn_logit_softcapping", - "use_bidirectional_attention", - "qk_scale_factor", - "use_qk_norm", - "use_attn_output_gate", - "output_multiplier", - "normalize_tok_embeddings", - "post_norm_eps", - "no_rope_layers", - } - ) - # flat vision field name -> canonical MuseGlimmerVisionConfig field name - _FLAT_VISION_RENAMES = { - "vision_latent_dim": "hidden_size", - "vision_heads": "num_attention_heads", - "vision_layers": "num_hidden_layers", - "vision_output_dim": "output_dim", - "vision_patch_size": "patch_size", - "vision_patch_temporal": "patch_temporal", - "vision_adapter_dim": "adapter_dim", - "vision_pos_emb_grid_h": "pos_emb_height", - "vision_pos_emb_grid_w": "pos_emb_width", - "vision_downsample_factor": "merge_kernel_size", - } - - @classmethod - def _looks_flat(cls, kwargs: dict) -> bool: - # Flat if there is no explicit text_config but there ARE text-level - # fields at the top level (e.g. hidden_size / num_hidden_layers). - if kwargs.get("text_config") is not None: - return False - return any(k in kwargs for k in ("hidden_size", "num_hidden_layers")) - - @classmethod - def _normalize_flat(cls, kwargs: dict) -> dict: - kwargs = dict(kwargs) - text: dict = {} - for key in list(kwargs.keys()): - if key in cls._FLAT_TEXT_KEYS: - canon = cls._FLAT_TEXT_RENAMES.get(key, key) - text.setdefault(canon, kwargs.pop(key)) - - vision_mlp_ratio = kwargs.pop("vision_mlp_ratio", None) - sparse_factor = kwargs.pop("vision_sparse_attention_factor", None) - vision: dict = {} - for key in list(kwargs.keys()): - if key in cls._FLAT_VISION_RENAMES: - vision.setdefault(cls._FLAT_VISION_RENAMES[key], kwargs.pop(key)) - - if vision_mlp_ratio is not None: - hidden_size = vision.get("hidden_size", 1536) - vision["intermediate_size"] = int(vision_mlp_ratio * hidden_size) - if sparse_factor is not None: - sparse_factor = int(sparse_factor) - if sparse_factor <= 0: - raise ValueError("vision_sparse_attention_factor must be positive") - num_layers = vision.get("num_hidden_layers", 50) - vision["layer_types"] = [ - "full_attention" - if (layer_idx + 1) % sparse_factor == 0 or layer_idx == num_layers - 1 - else "sliding_attention" - for layer_idx in range(num_layers) - ] - - if text: - kwargs["text_config"] = text - if vision: - kwargs["vision_config"] = vision - return kwargs - - def __init__( - self, - text_config: dict | MuseGlimmerTextConfig | None = None, - vision_config: dict | MuseGlimmerVisionConfig | None = None, - image_token_id: int = 200092, - video_token_id: int = 200091, - **kwargs, - ) -> None: - # Detect + fold a flat config into nested text/vision before building - # the sub-configs. (image/video token ids are handled below; a flat - # ``patch_token_id`` alias is mapped too.) - if text_config is None and self._looks_flat(kwargs): - kwargs = self._normalize_flat(kwargs) - text_config = kwargs.pop("text_config", None) - vision_config = kwargs.pop("vision_config", vision_config) - # flat image-token alias - if "patch_token_id" in kwargs and "image_token_id" not in kwargs: - image_token_id = kwargs.pop("patch_token_id") - - if text_config is None: - self.text_config = MuseGlimmerTextConfig() - elif isinstance(text_config, dict): - self.text_config = MuseGlimmerTextConfig(**text_config) - else: - self.text_config = text_config - - if vision_config is None: - self.vision_config = MuseGlimmerVisionConfig() - elif isinstance(vision_config, dict): - self.vision_config = MuseGlimmerVisionConfig(**vision_config) - else: - self.vision_config = vision_config - - self.image_token_id = image_token_id - self.video_token_id = video_token_id - self.patch_token_id = image_token_id - super().__init__(**kwargs) - - -class MuseGlimmerAssistantConfig(Qwen3Config): - """Config for the Muse Glimmer DFlash draft head. - - The head is Qwen3-shaped and runs on vLLM's generic ``qwen3_dflash`` - implementation, so this derives from ``Qwen3Config``. It cannot BE - ``Qwen3Config``, because two of the checkpoint's values do not survive that - class: - - * ``sliding_window`` is gated behind ``use_sliding_window``, which defaults - to False -- ``Qwen3Config(sliding_window=2048).sliding_window`` is None. - The checkpoint declares ``sliding_window: 2048`` and five - ``sliding_attention`` layers, so the window silently disappears and the - DFlash path then raises "sliding attention requires a window size". - * ``vocab_size`` is absent from the checkpoint, so Qwen3's default of - 151936 applies instead of Muse Glimmer's 202048. That one is *silent*: it - builds an all-zero ``draft_id_to_target_id`` remap, and also puts - pad/bos/eos/mask token ids out of range. - - Both defaults are set here so an unmodified checkpoint loads correctly. - """ - - model_type = "muse_glimmer_assistant" - - def __init__( - self, - vocab_size: int = 202048, - use_sliding_window: bool = True, - dflash_config: dict | None = None, - **kwargs, - ) -> None: - super().__init__( - vocab_size=vocab_size, use_sliding_window=use_sliding_window, **kwargs - ) - # muse_glimmer_assistant uses non-causal attention. - self.dflash_config = {"causal": False} | (dflash_config or {}) - - -__all__ = [ - "MuseGlimmerTextConfig", - "MuseGlimmerVisionConfig", - "MuseGlimmerConfig", - "MuseGlimmerAssistantConfig", -] diff --git a/vllm/transformers_utils/configs/openvla.py b/vllm/transformers_utils/configs/openvla.py deleted file mode 100644 index dba2f7f5a2..0000000000 --- a/vllm/transformers_utils/configs/openvla.py +++ /dev/null @@ -1,50 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""OpenVLA configuration support. - -OpenVLA checkpoints use a custom ``model_type`` and nest the language model -configuration under ``text_config``. This shim lets vLLM load the checkpoint -configuration without executing Hugging Face remote code. -""" - -from typing import Any - -from transformers import LlamaConfig, PreTrainedConfig - - -class OpenVLAConfig(PreTrainedConfig): - """Configuration class for OpenVLA models.""" - - model_type = "openvla" - - def __init__( - self, - timm_model_ids: list[str] | None = None, - timm_override_act_layers: list[str | None] | None = None, - image_sizes: list[int] | None = None, - use_fused_vision_backbone: bool = True, - image_token_index: int = 32000, - n_action_bins: int = 256, - text_config: dict[str, Any] | LlamaConfig | None = None, - **kwargs: Any, - ) -> None: - kwargs.setdefault("architectures", ["OpenVLAForActionPrediction"]) - super().__init__(**kwargs) - - self.timm_model_ids = timm_model_ids or [ - "vit_large_patch14_reg4_dinov2.lvd142m", - "vit_so400m_patch14_siglip_224", - ] - self.timm_override_act_layers = timm_override_act_layers or [None, None] - self.image_sizes = image_sizes or [224, 224] - self.use_fused_vision_backbone = use_fused_vision_backbone - self.image_token_index = image_token_index - self.n_action_bins = n_action_bins - - if text_config is None: - text_config = LlamaConfig(architectures=["LlamaForCausalLM"]) - elif isinstance(text_config, dict): - text_config = text_config.copy() - text_config.setdefault("architectures", ["LlamaForCausalLM"]) - text_config = LlamaConfig(**text_config) - self.text_config = text_config diff --git a/vllm/transformers_utils/configs/ovis.py b/vllm/transformers_utils/configs/ovis.py deleted file mode 100644 index d5662d7ef5..0000000000 --- a/vllm/transformers_utils/configs/ovis.py +++ /dev/null @@ -1,124 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -# ruff: noqa: E501 -# adapted from https://huggingface.co/AIDC-AI/Ovis2-1B/blob/main/configuration_ovis.py -from typing import Any - -from transformers import Aimv2VisionConfig, AutoConfig, PreTrainedConfig - - -# ---------------------------------------------------------------------- -# Visual Tokenizer Configuration -# ---------------------------------------------------------------------- -class BaseVisualTokenizerConfig(PreTrainedConfig): - def __init__( - self, - vocab_size=16384, - tokenize_function="softmax", - tau=1.0, - depths=None, - drop_cls_token=False, - backbone_config: PreTrainedConfig | dict | None = None, - hidden_stride: int = 1, - **kwargs, - ): - super().__init__(**kwargs) - self.vocab_size = vocab_size - self.tokenize_function = tokenize_function - self.tau = tau - if isinstance(depths, str): - depths = [int(x) for x in depths.split("|")] - self.depths = depths - self.backbone_kwargs = dict[str, Any]() - self.drop_cls_token = drop_cls_token - if backbone_config is not None: - assert isinstance(backbone_config, (PreTrainedConfig, dict)), ( - f"expect `backbone_config` to be instance of PreTrainedConfig or dict, but got {type(backbone_config)} type" - ) - if not isinstance(backbone_config, PreTrainedConfig): - model_type = backbone_config.pop("model_type") - if model_type == "aimv2": - backbone_config = Aimv2VisionConfig(**backbone_config) - else: - backbone_config = AutoConfig.for_model( - model_type, **backbone_config - ) - self.backbone_config = backbone_config - self.hidden_stride = hidden_stride - - -class Aimv2VisualTokenizerConfig(BaseVisualTokenizerConfig): - model_type = "aimv2_visual_tokenizer" - - def __init__(self, **kwargs): - super().__init__(**kwargs) - if self.drop_cls_token: - self.drop_cls_token = False - if self.depths: - assert len(self.depths) == 1 - self.backbone_kwargs["num_hidden_layers"] = self.depths[0] - - -class SiglipVisualTokenizerConfig(BaseVisualTokenizerConfig): - model_type = "siglip_visual_tokenizer" - - def __init__(self, **kwargs): - super().__init__(**kwargs) - if self.drop_cls_token: - self.drop_cls_token = False - if self.depths: - assert len(self.depths) == 1 - self.backbone_kwargs["num_hidden_layers"] = self.depths[0] - - -AutoConfig.register("siglip_visual_tokenizer", SiglipVisualTokenizerConfig) -AutoConfig.register("aimv2_visual_tokenizer", Aimv2VisualTokenizerConfig) - - -# ---------------------------------------------------------------------- -# Ovis Configuration -# ---------------------------------------------------------------------- -class OvisConfig(PreTrainedConfig): - model_type = "ovis" - - def __init__( - self, - llm_config: PreTrainedConfig | dict | None = None, - visual_tokenizer_config: PreTrainedConfig | dict | None = None, - multimodal_max_length=8192, - hidden_size=None, - conversation_formatter_class=None, - llm_attn_implementation=None, - disable_tie_weight=False, - **kwargs, - ): - super().__init__(**kwargs) - if llm_config is not None: - assert isinstance(llm_config, (PreTrainedConfig, dict)), ( - f"expect `llm_config` to be instance of PreTrainedConfig or dict, but got {type(llm_config)} type" - ) - if not isinstance(llm_config, PreTrainedConfig): - model_type = llm_config["model_type"] - llm_config.pop("model_type") - llm_config = AutoConfig.for_model(model_type, **llm_config) - - # map llm_config to text_config - self.text_config = llm_config - if visual_tokenizer_config is not None: - assert isinstance(visual_tokenizer_config, (PreTrainedConfig, dict)), ( - f"expect `visual_tokenizer_config` to be instance of PreTrainedConfig or dict, but got {type(visual_tokenizer_config)} type" - ) - if not isinstance(visual_tokenizer_config, PreTrainedConfig): - model_type = visual_tokenizer_config["model_type"] - visual_tokenizer_config.pop("model_type") - visual_tokenizer_config = AutoConfig.for_model( - model_type, **visual_tokenizer_config - ) - - self.visual_tokenizer_config = visual_tokenizer_config - self.multimodal_max_length = multimodal_max_length - self.hidden_size = hidden_size - self.conversation_formatter_class = conversation_formatter_class - self.llm_attn_implementation = llm_attn_implementation - self.disable_tie_weight = disable_tie_weight diff --git a/vllm/transformers_utils/configs/qianfan_ocr.py b/vllm/transformers_utils/configs/qianfan_ocr.py deleted file mode 100644 index 73df82c8fe..0000000000 --- a/vllm/transformers_utils/configs/qianfan_ocr.py +++ /dev/null @@ -1,105 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from typing import Any - -from transformers import PreTrainedConfig -from transformers.models.auto import CONFIG_MAPPING - - -class QianfanOCRVisionConfig(PreTrainedConfig): - model_type = "qianfan_ocr_vision" - - def __init__( - self, - hidden_size: int = 1024, - intermediate_size: int = 4096, - num_hidden_layers: int = 24, - num_attention_heads: int = 16, - num_channels: int = 3, - image_size: int = 448, - patch_size: int = 14, - hidden_act: str = "gelu", - layer_norm_eps: float = 1e-6, - attention_dropout: float = 0.0, - drop_path_rate: float = 0.1, - qkv_bias: bool = True, - qk_normalization: bool = False, - norm_type: str = "layer_norm", - initializer_range: float = 0.02, - initializer_factor: float = 0.1, - use_mask_token: bool = False, - use_mean_pooling: bool = True, - **kwargs: Any, - ): - super().__init__(**kwargs) - self.hidden_size = hidden_size - self.intermediate_size = intermediate_size - self.num_hidden_layers = num_hidden_layers - self.num_attention_heads = num_attention_heads - self.num_channels = num_channels - self.image_size = image_size - self.patch_size = patch_size - self.hidden_act = hidden_act - self.layer_norm_eps = layer_norm_eps - self.attention_dropout = attention_dropout - self.drop_path_rate = drop_path_rate - self.qkv_bias = qkv_bias - self.qk_normalization = qk_normalization - self.norm_type = norm_type - self.initializer_range = initializer_range - self.initializer_factor = initializer_factor - self.use_mask_token = use_mask_token - self.use_mean_pooling = use_mean_pooling - - -class QianfanOCRConfig(PreTrainedConfig): - model_type = "qianfan_ocr" - - def __init__( - self, - vision_config: dict | None = None, - text_config: dict | None = None, - downsample_ratio: float = 0.5, - dynamic_image_size: bool = True, - force_image_size: int = 448, - image_token_id: int = 151671, - max_dynamic_patch: int = 12, - min_dynamic_patch: int = 1, - pad2square: bool = False, - ps_version: str = "v2", - select_layer: int = -1, - template: str = "internvl2_5", - use_thumbnail: bool = True, - tie_word_embeddings: bool = False, - **kwargs: Any, - ): - super().__init__(**kwargs) - - if isinstance(vision_config, dict): - self.vision_config = QianfanOCRVisionConfig(**vision_config) - elif vision_config is None: - self.vision_config = QianfanOCRVisionConfig() - else: - self.vision_config = vision_config - - if isinstance(text_config, dict): - model_type = text_config.get("model_type", "qwen3") - self.text_config = CONFIG_MAPPING[model_type](**text_config) - elif text_config is None: - self.text_config = CONFIG_MAPPING["qwen3"]() - else: - self.text_config = text_config - - self.downsample_ratio = downsample_ratio - self.dynamic_image_size = dynamic_image_size - self.force_image_size = force_image_size - self.image_token_id = image_token_id - self.max_dynamic_patch = max_dynamic_patch - self.min_dynamic_patch = min_dynamic_patch - self.pad2square = pad2square - self.ps_version = ps_version - self.select_layer = select_layer - self.template = template - self.use_thumbnail = use_thumbnail - self.tie_word_embeddings = tie_word_embeddings diff --git a/vllm/transformers_utils/configs/qwen3_asr.py b/vllm/transformers_utils/configs/qwen3_asr.py deleted file mode 100644 index 77125e4b22..0000000000 --- a/vllm/transformers_utils/configs/qwen3_asr.py +++ /dev/null @@ -1,455 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -# ruff: noqa -# mypy: ignore-errors -# coding=utf-8 -# Copyright 2026 The Qwen team, Alibaba Group and the HuggingFace Inc. team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from transformers.configuration_utils import PreTrainedConfig -from transformers.modeling_rope_utils import rope_config_validation -from transformers.utils import logging - -logger = logging.get_logger(__name__) - - -class Qwen3ASRAudioEncoderConfig(PreTrainedConfig): - r""" - This is the configuration class to store the configuration of a [`Qwen3ASRAudioEncoder`]. It is used to instantiate a - Qwen3-ASR audio encoder according to the specified arguments, defining the model architecture. Instantiating a - configuration with the defaults will yield a similar configuration to that of the audio encoder of the Qwen2-Audio - architecture. - - e.g. [Qwen/Qwen3-ASR-1.7B](https://huggingface.co/Qwen/Qwen3-ASR-1.7B) - - Configuration objects inherit from [`PreTrainedConfig`] and can be used to control the model outputs. Read the - documentation from [`PreTrainedConfig`] for more information. - - Args: - num_mel_bins (`int`, *optional*, defaults to 128): - Number of mel features used per input features. Should correspond to the value used in the - `Qwen3ASRProcessor` class. - encoder_layers (`int`, *optional*, defaults to 32): - Number of encoder layers. - encoder_attention_heads (`int`, *optional*, defaults to 20): - Number of attention heads for each attention layer in the Transformer encoder. - encoder_ffn_dim (`int`, *optional*, defaults to 5120): - Dimensionality of the "intermediate" (often named feed-forward) layer in encoder. - d_model (`int`, *optional*, defaults to 1280): - Dimensionality of the layers. - dropout (`float`, *optional*, defaults to 0.0): - The dropout probability for all fully connected layers in the embeddings, encoder, and pooler. - attention_dropout (`float`, *optional*, defaults to 0.0): - The dropout ratio for the attention probabilities. - activation_function (`str`, *optional*, defaults to `"gelu"`): - The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`, - `"relu"`, `"silu"` and `"gelu_new"` are supported. - activation_dropout (`float`, *optional*, defaults to 0.0): - The dropout ratio for activations inside the fully connected layer. - scale_embedding (`bool`, *optional*, defaults to `False`): - Scale embeddings by diving by sqrt(d_model). - initializer_range (`float`, *optional*, defaults to 0.02): - The standard deviation of the truncated_normal_initializer for initializing all weight matrices. - max_source_positions (`int`, *optional*, defaults to 1500): - The maximum sequence length of log-mel filter-bank features that this model might ever be used with. - n_window (`int`, *optional*, defaults to 100): - The chunk for conv and flash attn in AudioEncoder. - output_dim (`int`, *optional*, defaults to 3584): - The output dimension of AudioEncoder. - - Example: - - ```python - >>> from transformers import Qwen3ASRAudioEncoderConfig, Qwen3ASRAudioEncoder - - >>> # Initializing a Qwen3ASRAudioEncoderConfig - >>> configuration = Qwen3ASRAudioEncoderConfig() - - >>> # Initializing a Qwen3ASRAudioEncoder (with random weights) - >>> model = Qwen3ASRAudioEncoder(configuration) - - >>> # Accessing the model configuration - >>> configuration = model.config - ```""" - - model_type = "qwen3_asr_audio_encoder" - - def __init__( - self, - num_mel_bins=128, - encoder_layers=32, - encoder_attention_heads=20, - encoder_ffn_dim=5120, - d_model=1280, - dropout=0, - attention_dropout=0, - activation_function="gelu", - activation_dropout=0, - scale_embedding=False, - initializer_range=0.02, - max_source_positions=1500, - n_window=100, - output_dim=3584, - n_window_infer=400, - conv_chunksize=500, - downsample_hidden_size=480, - **kwargs, - ): - super().__init__(**kwargs) - - self.num_mel_bins = num_mel_bins - self.d_model = d_model - self.encoder_layers = encoder_layers - self.encoder_attention_heads = encoder_attention_heads - self.encoder_ffn_dim = encoder_ffn_dim - self.dropout = dropout - self.attention_dropout = attention_dropout - self.activation_function = activation_function - self.activation_dropout = activation_dropout - self.num_hidden_layers = encoder_layers - self.initializer_range = initializer_range - self.scale_embedding = ( - scale_embedding # scale factor will be sqrt(d_model) if True - ) - self.max_source_positions = max_source_positions - self.n_window = n_window - self.output_dim = output_dim - self.n_window_infer = n_window_infer - self.conv_chunksize = conv_chunksize - self.downsample_hidden_size = downsample_hidden_size - - -class Qwen3ASRTextConfig(PreTrainedConfig): - r""" - This is the configuration class to store the configuration of a [`Qwen3ASRTextModel`]. It is used to instantiate a - Qwen3-ASR model according to the specified arguments, defining the model architecture. Instantiating a configuration - with the defaults will yield a similar configuration to that of - Qwen3-ASR-1.7B [Qwen/Qwen3-ASR-1.7B](https://huggingface.co/Qwen/Qwen3-ASR-1.7B) - - Configuration objects inherit from [`PreTrainedConfig`] and can be used to control the model outputs. Read the - documentation from [`PreTrainedConfig`] for more information. - - Args: - vocab_size (`int`, *optional*, defaults to 151936): - Vocabulary size of the Qwen3ASR model. Defines the number of different tokens that can be represented by the - `inputs_ids` passed when calling [`Qwen3ASRModel`] - hidden_size (`int`, *optional*, defaults to 4096): - Dimension of the hidden representations. - intermediate_size (`int`, *optional*, defaults to 22016): - Dimension of the MLP representations. - num_hidden_layers (`int`, *optional*, defaults to 32): - Number of hidden layers in the Transformer encoder. - num_attention_heads (`int`, *optional*, defaults to 32): - Number of attention heads for each attention layer in the Transformer encoder. - num_key_value_heads (`int`, *optional*, defaults to 32): - This is the number of key_value heads that should be used to implement Grouped Query Attention. If - `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if - `num_key_value_heads=1` the model will use Multi Query Attention (MQA) otherwise GQA is used. When - converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed - by meanpooling all the original heads within that group. For more details, check out [this - paper](https://huggingface.co/papers/2305.13245). If it is not specified, will default to `32`. - head_dim (`int`, *optional*, defaults to 128): - The dimension of the head. If not specified, will default to `hidden_size // num_attention_heads`. - hidden_act (`str` or `function`, *optional*, defaults to `"silu"`): - The non-linear activation function (function or string) in the decoder. - max_position_embeddings (`int`, *optional*, defaults to 128000): - The maximum sequence length that this model might ever be used with. - initializer_range (`float`, *optional*, defaults to 0.02): - The standard deviation of the truncated_normal_initializer for initializing all weight matrices. - rms_norm_eps (`float`, *optional*, defaults to 1e-06): - The epsilon used by the rms normalization layers. - use_cache (`bool`, *optional*, defaults to `True`): - Whether or not the model should return the last key/values attentions (not used by all models). Only - relevant if `config.is_decoder=True`. - tie_word_embeddings (`bool`, *optional*, defaults to `False`): - Whether the model's input and output word embeddings should be tied. - rope_theta (`float`, *optional*, defaults to 5000000.0): - The base period of the RoPE embeddings. - rope_scaling (`Dict`, *optional*): - Dictionary containing the scaling configuration for the RoPE embeddings. NOTE: if you apply new rope type - and you expect the model to work on longer `max_position_embeddings`, we recommend you to update this value - accordingly. - Expected contents: - `rope_type` (`str`): - The sub-variant of RoPE to use. Can be one of ['default', 'linear', 'dynamic', 'yarn', 'longrope', - 'llama3'], with 'default' being the original RoPE implementation. - `factor` (`float`, *optional*): - Used with all rope types except 'default'. The scaling factor to apply to the RoPE embeddings. In - most scaling types, a `factor` of x will enable the model to handle sequences of length x * - original maximum pre-trained length. - `original_max_position_embeddings` (`int`, *optional*): - Used with 'dynamic', 'longrope' and 'llama3'. The original max position embeddings used during - pretraining. - `attention_factor` (`float`, *optional*): - Used with 'yarn' and 'longrope'. The scaling factor to be applied on the attention - computation. If unspecified, it defaults to value recommended by the implementation, using the - `factor` field to infer the suggested value. - `beta_fast` (`float`, *optional*): - Only used with 'yarn'. Parameter to set the boundary for extrapolation (only) in the linear - ramp function. If unspecified, it defaults to 32. - `beta_slow` (`float`, *optional*): - Only used with 'yarn'. Parameter to set the boundary for interpolation (only) in the linear - ramp function. If unspecified, it defaults to 1. - `short_factor` (`list[float]`, *optional*): - Only used with 'longrope'. The scaling factor to be applied to short contexts (< - `original_max_position_embeddings`). Must be a list of numbers with the same length as the hidden - size divided by the number of attention heads divided by 2 - `long_factor` (`list[float]`, *optional*): - Only used with 'longrope'. The scaling factor to be applied to long contexts (< - `original_max_position_embeddings`). Must be a list of numbers with the same length as the hidden - size divided by the number of attention heads divided by 2 - `low_freq_factor` (`float`, *optional*): - Only used with 'llama3'. Scaling factor applied to low frequency components of the RoPE - `high_freq_factor` (`float`, *optional*): - Only used with 'llama3'. Scaling factor applied to high frequency components of the RoPE - attention_bias (`bool`, defaults to `False`, *optional*, defaults to `False`): - Whether to use a bias in the query, key, value and output projection layers during self-attention. - attention_dropout (`float`, *optional*, defaults to 0.0): - The dropout ratio for the attention probabilities. - - ```python - >>> from transformers import Qwen3ASRTextModel, Qwen3ASRTextConfig - - >>> # Initializing a Qwen3ASR style configuration - >>> configuration = Qwen3ASRTextConfig() - - >>> # Initializing a model from the Qwen3-VL-7B style configuration - >>> model = Qwen3ASRTextModel(configuration) - - >>> # Accessing the model configuration - >>> configuration = model.config - ```""" - - model_type = "qwen3_asr_text" - base_config_key = "text_config" - - def __init__( - self, - vocab_size=151936, - hidden_size=4096, - intermediate_size=22016, - num_hidden_layers=32, - num_attention_heads=32, - num_key_value_heads=32, - head_dim=128, - hidden_act="silu", - max_position_embeddings=128000, - initializer_range=0.02, - rms_norm_eps=1e-6, - use_cache=True, - tie_word_embeddings=False, - rope_theta=5000000.0, - rope_scaling=None, - attention_bias=False, - attention_dropout=0.0, - **kwargs, - ): - self.vocab_size = vocab_size - self.max_position_embeddings = max_position_embeddings - self.hidden_size = hidden_size - self.intermediate_size = intermediate_size - self.num_hidden_layers = num_hidden_layers - self.num_attention_heads = num_attention_heads - - # for backward compatibility - if num_key_value_heads is None: - num_key_value_heads = num_attention_heads - - self.num_key_value_heads = num_key_value_heads - self.head_dim = head_dim - self.hidden_act = hidden_act - self.initializer_range = initializer_range - self.rms_norm_eps = rms_norm_eps - self.use_cache = use_cache - self.rope_theta = rope_theta - self.rope_scaling = rope_scaling - self.attention_bias = attention_bias - self.attention_dropout = attention_dropout - # Validate the correctness of rotary position embeddings parameters - # BC: if there is a 'type' field, move it to 'rope_type'. - if self.rope_scaling is not None and "type" in self.rope_scaling: - self.rope_scaling["rope_type"] = self.rope_scaling["type"] - rope_config_validation(self) - - super().__init__(tie_word_embeddings=tie_word_embeddings, **kwargs) - - -class Qwen3ASRThinkerConfig(PreTrainedConfig): - r""" - This is the configuration class to store the configuration of a [`Qwen3ASRThinker`]. It is used to instantiate a - Qwen3-ASR-Thinker model according to the specified arguments, defining the model architecture. Instantiating a - configuration with the defaults will yield a similar configuration to that of the thinker component of the Qwen3-Omni - architecture. - - e.g. [Qwen/Qwen3-ASR-1.7B](https://huggingface.co/Qwen/Qwen3-ASR-1.7B) - - Configuration objects inherit from [`PreTrainedConfig`] and can be used to control the model outputs. Read the - documentation from [`PreTrainedConfig`] for more information. - - Args: - audio_config (`dict`, *optional*): - The config dictionary of the audio backbone. - text_config (`dict`, *optional*): - The config dictionary of the text backbone. - audio_token_id (`int`, *optional*, defaults to 151646): - The audio token id to encode the audio prompt. - audio_start_token_id (`int`, *optional*, defaults to 151647): - The audio start token id to encode the audio prompt. - user_token_id (`int`, *optional*, defaults to 872): - The user token id to encode the user token. - initializer_range (`float`, *optional*, defaults to 0.02): - The standard deviation of the truncated_normal_initializer for initializing all weight matrices. - - Example: - - ```python - >>> from transformers import Qwen3ASRThinkerModel, Qwen3ASRThinkerConfig - - >>> # Initializing a default Qwen3ASRThinkerConfig - >>> configuration = Qwen3ASRThinkerConfig() - - >>> # Initializing a model (with random weights) from the default configuration - >>> model = Qwen3ASRThinkerModel(configuration) - - >>> # Accessing the model configuration - >>> configuration = model.config - ```""" - - model_type = "qwen3_asr_thinker" - - attribute_map = {} - sub_configs = { - "audio_config": Qwen3ASRAudioEncoderConfig, - "text_config": Qwen3ASRTextConfig, - } - - def __init__( - self, - audio_config=None, - text_config=None, - audio_token_id=151646, - audio_start_token_id=151647, - user_token_id=872, - initializer_range=0.02, - classify_num=None, - **kwargs, - ): - super().__init__(**kwargs) - self.user_token_id = user_token_id - self.audio_start_token_id = audio_start_token_id - self.initializer_range = initializer_range - self.classify_num = classify_num - - if isinstance(audio_config, dict): - audio_config = Qwen3ASRAudioEncoderConfig(**audio_config) - elif audio_config is None: - audio_config = Qwen3ASRAudioEncoderConfig() - self.audio_config = audio_config - - if isinstance(text_config, dict): - text_config = Qwen3ASRTextConfig(**text_config) - elif text_config is None: - text_config = Qwen3ASRTextConfig() - self.text_config = text_config - self.audio_token_id = audio_token_id - - -class Qwen3ASRConfig(PreTrainedConfig): - """ - This is the configuration class to store the configuration of a [`Qwen3ASRForConditionalGeneration`]. It is used to instantiate a Qwen3ASR - model according to the specified sub-models configurations, defining the model architecture. - - Instantiating a configuration with the defaults will yield a similar configuration to that of the - [Qwen/Qwen3-ASR-1.7B](https://huggingface.co/Qwen/Qwen3-ASR-1.7B) architecture. - - Configuration objects inherit from [`PreTrainedConfig`] and can be used to control the model outputs. Read the - documentation from [`PreTrainedConfig`] for more information. - - Args: - thinker_config (`dict`, *optional*): Configuration of the underlying thinker sub-model. - support_languages (`List[str]`, *optional*): The languages supported by the model. - - Example: - - ```python - >>> from transformers import ( - ... Qwen3ASRThinkerConfig, - ... Qwen3ASRForConditionalGeneration, - ... Qwen3ASRConfig, - ... ) - - >>> # Initializing a Qwen3ASR style configuration - >>> configuration = Qwen3ASRConfig() - - >>> # Initializing a model from the configuration - >>> model = Qwen3ASRForConditionalGeneration(configuration) - - >>> # Accessing the model configuration - >>> configuration = model.config - ```""" - - model_type = "qwen3_asr" - sub_configs = { - "thinker_config": Qwen3ASRThinkerConfig, - } - - def __init__( - self, - thinker_config=None, - audio_config=None, - text_config=None, - audio_token_id=None, - support_languages=None, - timestamp_token_id=None, - timestamp_segment_time=None, - **kwargs, - ): - if thinker_config is None: - thinker_config = { - key: value - for key, value in ( - ("audio_config", audio_config), - ("text_config", text_config), - ("audio_token_id", audio_token_id), - ) - if value is not None - } - if not thinker_config: - logger.info( - "thinker_config is None. Initializing thinker model with " - "default values" - ) - - self.thinker_config = Qwen3ASRThinkerConfig(**thinker_config) - self.support_languages = support_languages - self.timestamp_token_id = timestamp_token_id - self.timestamp_segment_time = timestamp_segment_time - super().__init__(**kwargs) - - def get_text_config(self, decoder=False) -> "PreTrainedConfig": - """ - Returns the config that is meant to be used with text IO. On most models, it is the original config instance - itself. On specific composite models, it is under a set of valid names. - - Args: - decoder (`Optional[bool]`, *optional*, defaults to `False`): - If set to `True`, then only search for decoder config names. - """ - # Overridden for deeply nested config like Qwen2.5-Omni. We don't have any omni model - # except for Qwen yet. This has to be generalized if more deeply nested configs are - # added. NOTE: currently method used only by vLLM - return self.thinker_config.get_text_config() - - -__all__ = ["Qwen3ASRConfig", "Qwen3ASRThinkerConfig", "Qwen3ASRAudioEncoderConfig"] diff --git a/vllm/transformers_utils/configs/radio.py b/vllm/transformers_utils/configs/radio.py deleted file mode 100644 index 89a56a85ef..0000000000 --- a/vllm/transformers_utils/configs/radio.py +++ /dev/null @@ -1,110 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Radio vision model configuration.""" - -from typing import Any - -from transformers.configuration_utils import PreTrainedConfig -from transformers.utils import logging - -logger = logging.get_logger(__name__) - -VIT_TIMM_DIM_BY_NAME: dict[str, tuple[int, int, int, int]] = { - "vit_small_patch16_224": (384, 12, 6, 1536), - "vit_base_patch16_224": (768, 12, 12, 3072), - "vit_large_patch16_224": (1024, 24, 16, 4096), - "vit_huge_patch16_224": (1280, 32, 16, 5120), -} - -OPENAI_CLIP_MEAN = (0.48145466, 0.4578275, 0.40821073) -OPENAI_CLIP_STD = (0.26862954, 0.26130258, 0.27577711) - - -class RadioConfig(PreTrainedConfig): - r"""This is the configuration class to store the configuration of a Radio - vision model. It is used to instantiate a Radio model according to the - specified arguments, defining the model architecture. - - Args: - model_name: Name of the vision transformer model - (e.g., "vit_base_patch16_224"). Used to determine architecture - dimensions from `VIT_TIMM_DIM_BY_NAME`. - image_size: The size (resolution) of each image. - patch_size: The size (resolution) of each patch. - qkv_bias: Whether to add a bias to the queries, keys and values. - qk_normalization: Whether to apply normalization to queries and keys. - norm_type: The normalization type to use. - layer_norm_eps: The epsilon used by the layer normalization layers. - initializer_factor: A factor for initializing all weight matrices. - hidden_act: The non-linear activation function in the encoder. - cpe_max_size: Maximum image size for position embeddings. - norm_mean: Mean values for image normalization (RGB channels). - Defaults to (0.48145466, 0.4578275, 0.40821073)). - norm_std: Standard deviation values for image normalization - (RGB channels). Defaults to (0.26862954, 0.26130258, 0.27577711)). - register_multiple: Number of register tokens to use. - teachers: A list of teacher model configurations. Each teacher configuration is - a dict with keys like "name" and some may have "use_summary". - cls_token_per_teacher: Whether to use a separate CLS token for each teacher. - video_temporal_patch_size: Number of consecutive video frames grouped into - a single tubelet for temporal compression. Default 1 (no compression). - When > 1, a dedicated video_embedder (3*T*P*P -> hidden) is created - alongside the image embedder (3*P*P -> hidden). - separate_video_embedder: When True and video_temporal_patch_size > 1, use a - dedicated video patch embedder (3*T*P*P -> hidden) separate from the - image embedder (3*P*P -> hidden). When False, a single embedder with - input size 3*T*P*P is used for both (images are duplicated T times). - - """ - - model_type = "radio" - - def __init__( - self, - model_name: str, - image_size: int = 224, - patch_size: int = 16, - qkv_bias: bool = True, - qk_normalization: bool = False, - norm_type: str = "layer_norm", - layer_norm_eps: float = 1e-6, - initializer_factor: float = 1.0, - hidden_act: str = "gelu", - cpe_max_size: int = 2048, - norm_mean: tuple[float, float, float] | list = OPENAI_CLIP_MEAN, - norm_std: tuple[float, float, float] | list = OPENAI_CLIP_STD, - register_multiple: int | None = None, - teachers: list[dict[str, Any]] | None = None, - cls_token_per_teacher: bool = False, - video_temporal_patch_size: int = 1, - separate_video_embedder: bool = True, - **kwargs, - ): - self.model_name = model_name - ( - self.hidden_size, - self.num_hidden_layers, - self.num_attention_heads, - self.intermediate_size, - ) = VIT_TIMM_DIM_BY_NAME[model_name] - self.image_size = image_size - self.patch_size = patch_size - self.qkv_bias = qkv_bias - self.qk_normalization = qk_normalization - self.norm_type = norm_type - self.layer_norm_eps = layer_norm_eps - self.initializer_factor = initializer_factor - self.hidden_act = hidden_act - self.cpe_max_size = cpe_max_size - self.norm_mean = ( - list(norm_mean) if isinstance(norm_mean, (tuple, list)) else norm_mean - ) - self.norm_std = ( - list(norm_std) if isinstance(norm_std, (tuple, list)) else norm_std - ) - self.register_multiple = register_multiple - self.teachers = teachers if teachers is not None else [] - self.cls_token_per_teacher = cls_token_per_teacher - self.video_temporal_patch_size = video_temporal_patch_size - self.separate_video_embedder = separate_video_embedder - super().__init__(**kwargs) diff --git a/vllm/transformers_utils/configs/step3_vl.py b/vllm/transformers_utils/configs/step3_vl.py deleted file mode 100644 index c19030e554..0000000000 --- a/vllm/transformers_utils/configs/step3_vl.py +++ /dev/null @@ -1,178 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from typing import Any - -from transformers.configuration_utils import PreTrainedConfig - - -class Step3VisionEncoderConfig(PreTrainedConfig): - model_type = "step3_vision_encoder" - - def __init__( - self, - hidden_size=1792, - intermediate_size=3072, - output_hidden_size=4096, - num_hidden_layers=63, - num_attention_heads=16, - num_channels=3, - image_size=728, - patch_size=14, - hidden_act="quick_gelu", - layer_norm_eps=1e-5, - **kwargs, - ): - self.hidden_size = hidden_size - self.intermediate_size = intermediate_size - self.output_hidden_size = output_hidden_size - self.num_hidden_layers = num_hidden_layers - self.num_attention_heads = num_attention_heads - self.num_channels = num_channels - self.patch_size = patch_size - self.image_size = image_size - self.layer_norm_eps = layer_norm_eps - self.hidden_act = hidden_act - super().__init__(**kwargs) - - -class Step3TextConfig(PreTrainedConfig): - model_type = "step3_text" - architectures = ["Step3TextForCausalLM"] - - def __init__( - self, - hidden_size: int = 7168, - intermediate_size: int = 18432, - num_attention_heads: int = 64, - num_attention_groups: int = 1, - num_hidden_layers: int = 61, - max_seq_len: int = 65536, - vocab_size: int = 128815, - rms_norm_eps: float = 1e-5, - moe_intermediate_size: int = 5120, - moe_num_experts: int = 48, - moe_top_k: int = 3, - rope_parameters: dict[str, Any] | None = None, - max_position_embedding: int = 65536, - share_expert_dim: int = 5120, - share_q_dim: int = 2048, - head_dim: int = 256, - norm_expert_weight: bool = False, - moe_layers_enum: tuple[int, ...] = ( - 4, - 5, - 6, - 7, - 8, - 9, - 10, - 11, - 12, - 13, - 14, - 15, - 16, - 17, - 18, - 19, - 20, - 21, - 22, - 23, - 24, - 25, - 26, - 27, - 28, - 29, - 30, - 31, - 32, - 33, - 34, - 35, - 36, - 37, - 38, - 39, - 40, - 41, - 42, - 43, - 44, - 45, - 46, - 47, - 48, - 49, - 50, - 51, - 52, - 53, - 54, - 55, - 56, - 57, - 58, - 59, - ), - **kwargs, - ) -> None: - self.hidden_size = hidden_size - self.intermediate_size = intermediate_size - self.num_attention_heads = num_attention_heads - self.num_attention_groups = num_attention_groups - self.num_hidden_layers = num_hidden_layers - self.max_seq_len = max_seq_len - self.vocab_size = vocab_size - self.rms_norm_eps = rms_norm_eps - self.moe_intermediate_size = moe_intermediate_size - self.moe_num_experts = moe_num_experts - self.moe_top_k = moe_top_k - # Try to set `rope_scaling` if available, otherwise use `rope_parameters` - rope_scaling = kwargs.pop("rope_scaling", None) - rope_parameters = rope_scaling or rope_parameters or {"rope_type": "default"} - rope_theta = kwargs.pop("rope_theta", 500000.0) - if "rope_theta" not in rope_parameters: - rope_parameters["rope_theta"] = rope_theta - self.rope_parameters = rope_parameters - self.max_position_embedding = max_position_embedding - self.share_expert_dim = share_expert_dim - self.share_q_dim = share_q_dim - self.head_dim = head_dim - self.norm_expert_weight = norm_expert_weight - self.moe_layers_enum = moe_layers_enum - - super().__init__(**kwargs) - - -class Step3VLConfig(PreTrainedConfig): - model_type = "step3_vl" - - def __init__( - self, - vision_config: dict | Step3VisionEncoderConfig | None = None, - text_config: dict | Step3TextConfig | None = None, - understand_projector_stride: int = 1, - projector_bias: bool = True, - image_token_id: int = 128001, - **kwargs, - ) -> None: - if vision_config is None: - vision_config = Step3VisionEncoderConfig() - elif isinstance(vision_config, dict): - vision_config = Step3VisionEncoderConfig(**vision_config) - self.vision_config = vision_config - - if text_config is None: - text_config = Step3TextConfig() - elif isinstance(text_config, dict): - text_config = Step3TextConfig(**text_config) - self.text_config = text_config - - self.understand_projector_stride = understand_projector_stride - self.projector_bias = projector_bias - self.hidden_size = text_config.hidden_size - self.image_token_id = image_token_id - - super().__init__(**kwargs) diff --git a/vllm/transformers_utils/configs/step3p5.py b/vllm/transformers_utils/configs/step3p5.py deleted file mode 100644 index 1b8ac82a79..0000000000 --- a/vllm/transformers_utils/configs/step3p5.py +++ /dev/null @@ -1,103 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from typing import Any - -from transformers.configuration_utils import PreTrainedConfig - - -class Step3p5Config(PreTrainedConfig): - model_type = "step3p5" - - def __init__( - self, - hidden_size: int = 5120, - intermediate_size: int = 13312, - num_attention_heads: int = 40, - num_attention_groups: int = 8, - num_hidden_layers: int = 48, - max_seq_len: int = 4096, - vocab_size: int = 65536, - rms_norm_eps: float = 1e-5, - moe_every_n_layer: int = 2, - use_moe: bool = False, - moe_intermediate_size: int = 10240, - moe_num_experts: int = 16, - moe_top_k: int = 4, - moe_layer_offset: int = 0, - rope_theta: float | list[float] | None = 500000, - rope_scaling: dict[str, Any] | None = None, - head_dim: int | None = None, - share_expert_dim: int | None = None, - norm_expert_weight: bool = True, - bos_token_id: list[int] | int | None = None, - eos_token_id: list[int] | int | None = None, - moe_router_activation: str = "softmax", - moe_router_scaling_factor: float = 1.0, - att_impl_type: str = "GQA", - use_head_wise_attn_gate: bool = False, - use_moe_router_bias: bool = True, - need_fp32_gate: bool = True, - layer_types: list[str] | None = None, - use_rope_layers: list[bool] | None = None, - yarn_only_types: list[str] | None = None, - attention_other_setting: dict[str, Any] | None = None, - num_nextn_predict_layers: int = 0, - swiglu_limits: list[float] | None = None, - swiglu_limits_shared: list[float] | None = None, - max_position_embeddings: int | None = None, - **kwargs, - ): - self.hidden_size = hidden_size - self.intermediate_size = intermediate_size - self.num_attention_heads = num_attention_heads - self.num_attention_groups = num_attention_groups - self.num_hidden_layers = num_hidden_layers - self.max_seq_len = max_seq_len - self.vocab_size = vocab_size - self.rms_norm_eps = rms_norm_eps - self.use_moe = use_moe - self.moe_intermediate_size = moe_intermediate_size - self.moe_every_n_layer = moe_every_n_layer - self.moe_num_experts = moe_num_experts - self.num_experts_per_tok = moe_top_k - self.moe_top_k = moe_top_k - self.moe_layer_offset = moe_layer_offset - - self.rope_theta = rope_theta - self.rope_scaling = rope_scaling - self.head_dim = head_dim - if share_expert_dim is None: - self.share_expert_dim = self.moe_intermediate_size * self.moe_top_k - else: - self.share_expert_dim = share_expert_dim - self.norm_expert_weight = norm_expert_weight - - self.max_position_embeddings = max_position_embeddings - self.moe_router_activation = moe_router_activation - self.moe_router_scaling_factor = moe_router_scaling_factor - self.use_moe_router_bias = use_moe_router_bias - self.need_fp32_gate = need_fp32_gate - - self.att_impl_type = att_impl_type - self.use_head_wise_attn_gate = use_head_wise_attn_gate - # For some reason the checkpoint has longer layer_types than num_hidden_layers - if layer_types is not None: - layer_types = layer_types[: self.num_hidden_layers] - self.layer_types = layer_types - self.use_rope_layers = use_rope_layers - self.yarn_only_types = yarn_only_types - self.attention_other_setting = attention_other_setting - self.num_nextn_predict_layers = num_nextn_predict_layers - self.swiglu_limits = swiglu_limits - self.swiglu_limits_shared = swiglu_limits_shared - - resolved_bos_token_id = 1 if bos_token_id is None else bos_token_id - resolved_eos_token_id = [2, 3] if eos_token_id is None else eos_token_id - self.bos_token_id = resolved_bos_token_id - self.eos_token_id = resolved_eos_token_id - - super().__init__( - bos_token_id=resolved_bos_token_id, - eos_token_id=resolved_eos_token_id, - **kwargs, - ) diff --git a/vllm/transformers_utils/configs/ultravox.py b/vllm/transformers_utils/configs/ultravox.py deleted file mode 100644 index 669fe7be48..0000000000 --- a/vllm/transformers_utils/configs/ultravox.py +++ /dev/null @@ -1,120 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -# Adapted from https://github.com/fixie-ai/ultravox/blob/ecd58c4041030bae2ad15aa6bcf04ab43199ea02/ultravox/model/ultravox_config.py -from typing import Any - -import transformers - - -class UltravoxConfig(transformers.PreTrainedConfig): - r"""This is the configuration class to store the configuration of a - [`UltravoxForConditionalGeneration`]. It is used to instantiate an - Ultravox model according to the specified arguments, defining the model - architecture. - - Configuration objects inherit from [`PreTrainedConfig`] and can be used to - control the model outputs. Read the documentation from [`PreTrainedConfig`] - for more information. - - Args: - audio_config (`Union[AutoConfig, dict]`, *optional*): - Custom audio config or dict. - text_config (`Union[AutoConfig, dict]`, *optional*): - The config object of the text backbone. - audio_model_id (`str`, *optional*): - The model ID of the audio backbone. - text_model_id (`str`, *optional*): - The model ID of the text backbone. - ignore_index (`int`, *optional*, defaults to -100): - The ignore index for the loss function. - audio_token_index (`int`, *optional*, defaults to 32000): - The audio token index to encode the audio prompt. - stack_factor (`int`, *optional*, defaults to 8): - Audio downsampling factor for the multimodal projector. - norm_init (`float`, *optional*, defaults to 0.4): - The initialization value for the layer normalization. - projector_act (`str`, *optional*, defaults to `"swiglu"`): - The activation function used by the multimodal projector. - projector_ln_mid (`bool`, *optional*, defaults to `False`): - Whether to apply layer normalization at the middle of the - projector or at the end. Versions v0.4.1 and below - use `False`, but v0.5 and above use `True`. - - """ - - model_type = "ultravox" - audio_token = "<|audio|>" - is_composition = False - - def __init__( - self, - audio_config: dict[str, Any] | None = None, - text_config: dict[str, Any] | None = None, - audio_model_id: str | None = None, - text_model_id: str | None = None, - ignore_index: int = -100, - audio_token_index: int = 32000, - hidden_size: int = 4096, - stack_factor: int = 8, - norm_init: float = 0.4, - projector_act: str = "swiglu", - projector_ln_mid: bool = False, - num_projector_layers: int = 0, - **kwargs, - ): - self.ignore_index = ignore_index - self.audio_token_index = audio_token_index - - self.hidden_size = hidden_size - self.stack_factor = stack_factor - self.norm_init = norm_init - self.projector_act = projector_act - self.projector_ln_mid = projector_ln_mid - self.num_projector_layers = num_projector_layers - - # N.B. May set the wrapped_model_config below. - self.wrapped_model_config: transformers.PreTrainedConfig - self.text_model_id = text_model_id - if text_model_id is None: - text_config = text_config or {} - self.wrapped_model_config = transformers.CONFIG_MAPPING[ - text_config.get("model_type", "llama") - ](**text_config) - - # N.B. May set the audio_config below. - self.audio_model_id = audio_model_id - if audio_model_id is None: - self.audio_model_id = None - audio_config = audio_config or {} - self.audio_config = transformers.CONFIG_MAPPING[ - audio_config.get("model_type", "whisper") - ](**audio_config) - - super().__init__(**kwargs) - - def __setattr__(self, key, value): - # Since --hf-overrides are applied _after_ the UltravoxConfig is - # instantiated, load the configs implicitly when assigning text_model_id - # or audio_model_id. This allows: - # - # --hf-overrides.text_model_id= - # - # to behave as intended. - if key == "text_model_id" and value is not None: - from vllm.transformers_utils.config import get_config - - self.wrapped_model_config = get_config(value, trust_remote_code=False) - elif key == "audio_model_id" and value is not None: - from vllm.transformers_utils.config import get_config - - self.audio_config = get_config(value, trust_remote_code=False) - - return super().__setattr__(key, value) - - @property - def text_config(self) -> transformers.PreTrainedConfig: - # When Ultravox wraps a multi-modal model (e.g. Gemma), we instantiate - # the full model, but the text config is the text config of the inner - # model. - return self.wrapped_model_config.get_text_config() diff --git a/vllm/transformers_utils/configs/unlimited_ocr.py b/vllm/transformers_utils/configs/unlimited_ocr.py deleted file mode 100644 index 99e50a03c7..0000000000 --- a/vllm/transformers_utils/configs/unlimited_ocr.py +++ /dev/null @@ -1,35 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -# Unlimited-OCR (baidu/Unlimited-OCR) reuses -# the DeepSeek-OCR multimodal layout (DeepEncoder = SAM-ViT-B + CLIP-L, a linear -# MLP projector and a DeepSeek-V2 text backbone). The only architectural -# difference is the language model, which is a DeepSeek-V2 *MoE* with plain -# multi-head attention (``use_mla=False``) instead of the dense MLA backbone. -# We therefore reuse ``DeepseekVLV2Config`` for parsing the nested config. - -from vllm.transformers_utils.configs.deepseek_vl2 import DeepseekVLV2Config - - -class UnlimitedOCRConfig(DeepseekVLV2Config): - model_type = "unlimited-ocr" - - # An explicit ``__init__`` is required: Transformers v5 processes each - # concrete config class' ``__init__`` signature to build nested sub-configs, - # and an empty subclass (only overriding ``model_type``) would skip - # ``DeepseekVLV2Config.__init__``, leaving ``text_config`` unset. - def __init__( - self, - tile_tag: str = "2D", - global_view_pos: str = "head", - candidate_resolutions: tuple[tuple[int, int]] = ((384, 384),), - rswa_window: int = 128, - **kwargs, - ): - super().__init__( - tile_tag=tile_tag, - global_view_pos=global_view_pos, - candidate_resolutions=candidate_resolutions, - **kwargs, - ) - self.rswa_window = rswa_window diff --git a/vllm/transformers_utils/processors/__init__.py b/vllm/transformers_utils/processors/__init__.py index c06519ef18..4232835c98 100644 --- a/vllm/transformers_utils/processors/__init__.py +++ b/vllm/transformers_utils/processors/__init__.py @@ -41,37 +41,7 @@ __all__ = [ "Step3VLProcessor", ] -_CLASS_TO_MODULE: dict[str, str] = { - "BagelProcessor": "vllm.transformers_utils.processors.bagel", - "CohereASRProcessor": "vllm.transformers_utils.processors.cohere_asr", - "Cosmos3EdgeProcessor": "vllm.transformers_utils.processors.cosmos3_edge", - "DeepseekVLV2Processor": "vllm.transformers_utils.processors.deepseek_vl2", - "FireRedASR2Processor": "vllm.transformers_utils.processors.fireredasr2", - "FunASRProcessor": "vllm.transformers_utils.processors.funasr", - "GLM4VProcessor": "vllm.transformers_utils.processors.glm4v", - "H2OVLProcessor": "vllm.transformers_utils.processors.h2ovl", - "IsaacProcessor": "vllm.transformers_utils.processors.isaac", - "KimiAudioProcessor": "vllm.transformers_utils.processors.kimi_audio", - "KimiK25Processor": "vllm.transformers_utils.processors.kimi_k25", - "KimiK3Processor": "vllm.transformers_utils.processors.kimi_k3", - "MiMoOmniProcessor": "vllm.transformers_utils.processors.mimo_v2_omni", - "MiniCPMOProcessor": "vllm.transformers_utils.processors.minicpmo", - "MiniCPMVProcessor": "vllm.transformers_utils.processors.minicpmv", - "MiniMaxM3VLImageProcessor": "vllm.transformers_utils.processors.minimax_m3", - "MiniMaxM3VLVideoProcessor": "vllm.transformers_utils.processors.minimax_m3", - "MiniMaxVLProcessor": "vllm.transformers_utils.processors.minimax_m3", - "MistralCommonPixtralProcessor": "vllm.transformers_utils.processors.pixtral", - "MistralCommonVoxtralProcessor": "vllm.transformers_utils.processors.voxtral", - "Moondream3Processor": "vllm.transformers_utils.processors.moondream3", - "NanoNemotronVLProcessor": "vllm.transformers_utils.processors.nano_nemotron_vl", - "NemotronVLProcessor": "vllm.transformers_utils.processors.nemotron_vl", - "LlamaNemotronVLEmbedProcessor": "vllm.transformers_utils.processors.nemotron_vl", - "NVLMProcessor": "vllm.transformers_utils.processors.nvlm_d", - "OpenVLAProcessor": "vllm.transformers_utils.processors.openvla", - "OvisProcessor": "vllm.transformers_utils.processors.ovis", - "Ovis2_5Processor": "vllm.transformers_utils.processors.ovis2_5", - "Step3VLProcessor": "vllm.transformers_utils.processors.step3_vl", -} +_CLASS_TO_MODULE: dict[str, str] = {} def __getattr__(name: str): diff --git a/vllm/transformers_utils/processors/bagel.py b/vllm/transformers_utils/processors/bagel.py deleted file mode 100644 index b6fefe0327..0000000000 --- a/vllm/transformers_utils/processors/bagel.py +++ /dev/null @@ -1,73 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -# Copyright 2025 Bytedance Ltd. and/or its affiliates. -"""BAGEL processor for image and text inputs.""" - -from transformers.feature_extraction_utils import BatchFeature -from transformers.image_utils import ImageInput -from transformers.processing_utils import ProcessingKwargs, ProcessorMixin, Unpack -from transformers.tokenization_utils_base import PreTokenizedInput, TextInput - - -class BagelProcessorKwargs(ProcessingKwargs, total=False): # type: ignore[call-arg] - _defaults = { - "images_kwargs": { - "return_tensors": "pt", - }, - } - - -class BagelProcessor(ProcessorMixin): - """Constructs a BAGEL processor which wraps a - SigLIP image processor and a Qwen2 tokenizer. - """ - - attributes = ["image_processor", "tokenizer"] - image_processor_class = "SiglipImageProcessor" - tokenizer_class = "AutoTokenizer" - - def __call__( - self, - text: TextInput - | PreTokenizedInput - | list[TextInput] - | list[PreTokenizedInput] = None, - images: ImageInput = None, - **kwargs: Unpack[BagelProcessorKwargs], - ): - """Prepare one or more sequences and images for the model.""" - output_kwargs = self._merge_kwargs( - BagelProcessorKwargs, - tokenizer_init_kwargs=self.tokenizer.init_kwargs, - **kwargs, - ) - - if images is not None: - # Process images with the image processor - pixel_values = self.image_processor( - images, **output_kwargs["images_kwargs"] - ) - else: - pixel_values = {} - - text_inputs = ( - self.tokenizer(text, **output_kwargs["text_kwargs"]) - if text is not None - else {} - ) - - return BatchFeature(data={**pixel_values, **text_inputs}) - - def batch_decode(self, *args, **kwargs): - """Forward all arguments to `Qwen2TokenizerFast.batch_decode`.""" - return self.tokenizer.batch_decode(*args, **kwargs) - - def decode(self, *args, **kwargs): - """This method forwards all its arguments to Qwen2TokenizerFast's decode.""" - return self.tokenizer.decode(*args, **kwargs) - - @property - def model_input_names(self): - tokenizer_input_names = self.tokenizer.model_input_names - image_processor_input_names = self.image_processor.model_input_names - return list(dict.fromkeys(tokenizer_input_names + image_processor_input_names)) diff --git a/vllm/transformers_utils/processors/cohere_asr.py b/vllm/transformers_utils/processors/cohere_asr.py deleted file mode 100644 index 5665d3cbae..0000000000 --- a/vllm/transformers_utils/processors/cohere_asr.py +++ /dev/null @@ -1,573 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -import logging -import math -import random - -import numpy as np -import torch -import torch.nn.functional as F -from torch import nn -from torchaudio.functional import melscale_fbanks -from transformers import AutoFeatureExtractor, AutoProcessor, BatchFeature -from transformers.feature_extraction_sequence_utils import ( - SequenceFeatureExtractor, -) -from transformers.processing_utils import ProcessorMixin - -logger = logging.getLogger(__name__) - -CONSTANT = 1e-5 -INF_VAL = 10000.0 - - -class FilterbankFeatures(nn.Module): - """Featurizer that converts wavs to Mel Spectrograms. - See AudioToMelSpectrogramPreprocessor for args. - """ - - window: torch.Tensor - fb: torch.Tensor - - def __init__( - self, - sample_rate=16000, - n_window_size=320, - n_window_stride=160, - window="hann", - normalize="per_feature", - n_fft=None, - preemph=0.97, - nfilt=64, - lowfreq=0, - highfreq=None, - log=True, - log_zero_guard_type="add", - log_zero_guard_value=2**-24, - dither=CONSTANT, - pad_to=16, - max_duration=30, - frame_splicing=1, - exact_pad=False, - pad_value=0, - mag_power=2.0, - use_grads=False, - rng=None, - nb_augmentation_prob=0.0, - nb_max_freq=4000, - mel_norm="slaney", - stft_exact_pad=False, - stft_conv=False, - device="cpu", - ): - super().__init__() - if stft_conv or stft_exact_pad: - logger.warning( - "Using torch_stft is deprecated and has been removed. " - "The values have been forcibly set to False for " - "FilterbankFeatures and AudioToMelSpectrogramPreprocessor. " - "Please set exact_pad to True as needed." - ) - if exact_pad and n_window_stride % 2 == 1: - raise NotImplementedError( - f"{self} received exact_pad == True, but hop_size was odd. " - "If audio_length % hop_size == 0, the returned spectrogram " - "would not be of length audio_length // hop_size. " - "Please use an even hop_size." - ) - self.log_zero_guard_value = log_zero_guard_value - if ( - n_window_size is None - or n_window_stride is None - or not isinstance(n_window_size, int) - or not isinstance(n_window_stride, int) - or n_window_size <= 0 - or n_window_stride <= 0 - ): - raise ValueError( - f"{self} got an invalid value for either n_window_size or " - f"n_window_stride. Both must be positive ints." - ) - - self.sample_rate = sample_rate - self.win_length = n_window_size - self.hop_length = n_window_stride - self.n_fft = n_fft or 2 ** math.ceil(math.log2(self.win_length)) - self.stft_pad_amount = ( - (self.n_fft - self.hop_length) // 2 if exact_pad else None - ) - self.exact_pad = exact_pad - self.sample_rate = sample_rate - self.max_duration = max_duration - - if exact_pad: - logger.info("STFT using exact pad") - torch_windows = { - "hann": torch.hann_window, - "hamming": torch.hamming_window, - "blackman": torch.blackman_window, - "bartlett": torch.bartlett_window, - "none": None, - } - window_fn = torch_windows.get(window) - window_tensor = ( - window_fn(self.win_length, periodic=False) if window_fn else None - ) - self.register_buffer("window", window_tensor) - - self.normalize = normalize - self.log = log - self.dither = dither - self.frame_splicing = frame_splicing - self.nfilt = nfilt - self.preemph = preemph - self.pad_to = pad_to - highfreq = highfreq or sample_rate / 2 - self.sample_rate = sample_rate - # disable pad min duration - # self.pad_min_duration = 1.0 - self.pad_min_duration = 0.0 - self.pad_direction = "both" - - filterbanks = melscale_fbanks( - n_freqs=self.n_fft // 2 + 1, - f_min=lowfreq, - f_max=highfreq, - n_mels=nfilt, - sample_rate=sample_rate, - norm=mel_norm, - mel_scale="slaney", - ).T.unsqueeze(0) - self.register_buffer("fb", filterbanks) - - # Calculate maximum sequence length - max_length = self.get_seq_len( - torch.tensor(max_duration * sample_rate, dtype=torch.float) - ) - max_pad = pad_to - (max_length % pad_to) if pad_to > 0 else 0 - self.max_length = max_length + max_pad - self.pad_value = pad_value - self.mag_power = mag_power - - # We want to avoid taking the log of zero - # There are two options: either adding or clamping to a small value - if log_zero_guard_type not in ["add", "clamp"]: - raise ValueError( - f"{self} received {log_zero_guard_type} for the " - f"log_zero_guard_type parameter. It must be either 'add' or " - f"'clamp'." - ) - - self.use_grads = use_grads - if not use_grads: - self.forward = torch.no_grad()(self.forward) - self._rng = random.Random() if rng is None else rng - self.nb_augmentation_prob = nb_augmentation_prob - if self.nb_augmentation_prob > 0.0: - if nb_max_freq >= sample_rate / 2: - self.nb_augmentation_prob = 0.0 - else: - self._nb_max_fft_bin = int((nb_max_freq / sample_rate) * n_fft) - - # log_zero_guard_value is the the small we want to use, we support - # an actual number, or "tiny", or "eps" - self.log_zero_guard_type = log_zero_guard_type - - assert self.window is not None - assert self.fb is not None - self.window = self.window.to(dtype=torch.bfloat16) - self.fb = self.fb.to(dtype=torch.bfloat16) - - self.generator = torch.Generator(device=device) - self.generator.manual_seed(0) - - @torch._dynamo.disable - def stft(self, x): - # disable autocast to get full range of stft values - with torch.amp.autocast(x.device.type, enabled=False): - return torch.stft( - x, - n_fft=self.n_fft, - hop_length=self.hop_length, - win_length=self.win_length, - center=not self.exact_pad, - window=self.window.to(dtype=torch.float, device=x.device), - return_complex=True, - pad_mode="constant", - ) - - def log_zero_guard_value_fn(self, x): - if isinstance(self.log_zero_guard_value, str): - if self.log_zero_guard_value == "tiny": - return torch.finfo(x.dtype).tiny - elif self.log_zero_guard_value == "eps": - return torch.finfo(x.dtype).eps - else: - raise ValueError( - f"{self} received {self.log_zero_guard_value} for the " - f"log_zero_guard_type parameter. It must be either a " - f"number, 'tiny', or 'eps'" - ) - else: - return self.log_zero_guard_value - - def get_seq_len(self, seq_len): - # Assuming that center is True is stft_pad_amount = 0 - pad_amount = ( - self.stft_pad_amount * 2 - if self.stft_pad_amount is not None - else self.n_fft // 2 * 2 - ) - seq_len = torch.floor_divide( - (seq_len + pad_amount - self.n_fft), self.hop_length - ) - return seq_len.to(dtype=torch.long) - - @property - def filter_banks(self): - return self.fb - - def splice_frames(self, x, frame_splicing): - """Stacks frames together across feature dim. - - input is batch_size, feature_dim, num_frames - output is batch_size, feature_dim*frame_splicing, num_frames - - """ - seq = [x] - for n in range(1, frame_splicing): - seq.append(torch.cat([x[:, :, :n], x[:, :, n:]], dim=2)) - return torch.cat(seq, dim=1) - - def normalize_batch(self, x, seq_len, normalize_type): - x_mean = None - x_std = None - if normalize_type == "per_feature": - batch_size = x.shape[0] - max_time = x.shape[2] - - # When doing stream capture to a graph, item() is not allowed - # because it calls cudaStreamSynchronize(). Therefore, we are - # sacrificing some error checking when running with cuda graphs. - # if ( - # torch.cuda.is_available() - # and not torch.cuda.is_current_stream_capturing() - # and torch.any(seq_len == 1).item() - # ): - # raise ValueError( - # "normalize_batch with `per_feature` normalize_type " - # "received a tensor of length 1. This will result in " - # "torch.std() returning nan. Make sure your audio length " - # "has enough samples for a single feature (ex. at least " - # "`hop_length` for Mel Spectrograms)." - # ) - time_steps = ( - torch.arange(max_time, device=x.device) - .unsqueeze(0) - .expand(batch_size, max_time) - ) - valid_mask = time_steps < seq_len.unsqueeze(1) - x_mean_numerator = torch.where(valid_mask.unsqueeze(1), x, 0.0).sum(axis=2) - x_mean_denominator = valid_mask.sum(axis=1) - x_mean = x_mean_numerator / x_mean_denominator.unsqueeze(1) - - # Subtract 1 in the denominator to correct for the bias. - x_std = torch.sqrt( - torch.sum( - torch.where(valid_mask.unsqueeze(1), x - x_mean.unsqueeze(2), 0.0) - ** 2, - axis=2, - ) - / (x_mean_denominator.unsqueeze(1) - 1.0) - ) - x_std = x_std.masked_fill( - x_std.isnan(), 0.0 - ) # edge case: only 1 frame in denominator - # make sure x_std is not zero - x_std += CONSTANT - return (x - x_mean.unsqueeze(2)) / x_std.unsqueeze(2), x_mean, x_std - elif normalize_type == "all_features": - x_mean = torch.zeros(seq_len.shape, dtype=x.dtype, device=x.device) - x_std = torch.zeros(seq_len.shape, dtype=x.dtype, device=x.device) - for i in range(x.shape[0]): - x_mean[i] = x[i, :, : seq_len[i].item()].mean() - x_std[i] = x[i, :, : seq_len[i].item()].std() - # make sure x_std is not zero - x_std += CONSTANT - return (x - x_mean.view(-1, 1, 1)) / x_std.view(-1, 1, 1), x_mean, x_std - elif "fixed_mean" in normalize_type and "fixed_std" in normalize_type: - x_mean = torch.tensor(normalize_type["fixed_mean"], device=x.device) - x_std = torch.tensor(normalize_type["fixed_std"], device=x.device) - return ( - (x - x_mean.view(x.shape[0], x.shape[1]).unsqueeze(2)) - / x_std.view(x.shape[0], x.shape[1]).unsqueeze(2), - x_mean, - x_std, - ) - else: - return x, x_mean, x_std - - @torch.compile - def forward(self, x, seq_len, linear_spec=False): - if x.shape[1] < self.sample_rate * self.pad_min_duration: - pad_amount = int(self.sample_rate * self.pad_min_duration) - x.shape[1] - if self.pad_direction == "right": - x = F.pad(x, (0, pad_amount), value=self.pad_value) - elif self.pad_direction == "left": - x = F.pad(x, (pad_amount, 0), value=self.pad_value) - elif self.pad_direction == "both": - left_pad = pad_amount // 2 - right_pad = pad_amount - left_pad - x = F.pad(x, (left_pad, right_pad), value=self.pad_value) - else: - raise ValueError( - f"{self} received an invalid pad_direction: {self.pad_direction}. " - f"It must be one of 'left', 'right', or 'both'." - ) - seq_len = torch.tensor([x.shape[1]], dtype=torch.float, device=x.device) - - seq_len_time = seq_len - seq_len_unfixed = self.get_seq_len(seq_len) - - # fix for seq_len = 0 for streaming; if size was 0, it is always padded - # to 1, and normalizer fails - seq_len = torch.where( - seq_len == 0, torch.zeros_like(seq_len_unfixed), seq_len_unfixed - ) - - if self.stft_pad_amount is not None: - x = torch.nn.functional.pad( - x.unsqueeze(1), (self.stft_pad_amount, self.stft_pad_amount), "constant" - ).squeeze(1) - - # use dither for inference as well - if self.dither > 0: - x += self.dither * torch.randn( - x.shape, dtype=x.dtype, device=x.device, generator=self.generator - ) - - # do preemphasis - if self.preemph is not None: - timemask = torch.arange(x.shape[1], device=x.device).unsqueeze( - 0 - ) < seq_len_time.unsqueeze(1) - x = torch.cat( - (x[:, 0].unsqueeze(1), x[:, 1:] - self.preemph * x[:, :-1]), dim=1 - ) - - x = x.masked_fill(~timemask, 0.0) - - x = self.stft(x) - - # torch stft returns complex tensor (of shape [B,N,T]); so convert to magnitude - # guard is needed for sqrt if grads are passed through - guard = 0 if not self.use_grads else CONSTANT - x = torch.view_as_real(x) - x = torch.sqrt(x.pow(2).sum(-1) + guard) - - # get power spectrum - if self.mag_power != 1.0: - x = x.pow(self.mag_power) - - # return plain spectrogram if required - if linear_spec: - return x, seq_len - - # disable autocast, otherwise it might be automatically casted to fp16 - # on fp16 compatible GPUs and get NaN values for input value of 65520 - with torch.amp.autocast(x.device.type, enabled=False): - # dot with filterbank energies - x = torch.matmul(self.fb.to(x.dtype), x) - - # log features if required - if self.log: - if self.log_zero_guard_type == "add": - x = torch.log(x + self.log_zero_guard_value_fn(x)) - elif self.log_zero_guard_type == "clamp": - x = torch.log(torch.clamp(x, min=self.log_zero_guard_value_fn(x))) - else: - raise ValueError("log_zero_guard_type was not understood") - - # frame splicing if required - if self.frame_splicing > 1: - x = self.splice_frames(x, self.frame_splicing) - - # normalize if required - if self.normalize: - x, _, _ = self.normalize_batch(x, seq_len, normalize_type=self.normalize) - - # mask to zero any values beyond seq_len in batch, pad to multiple of - # `pad_to` (for efficiency) - max_len = x.size(-1) - mask = torch.arange(max_len, device=x.device) - mask = mask.repeat(x.size(0), 1) >= seq_len.unsqueeze(1) - x = x.masked_fill( - mask.unsqueeze(1).type(torch.bool).to(device=x.device), self.pad_value - ) - - del mask - pad_to = self.pad_to - if pad_to == "max": - x = nn.functional.pad( - x, (0, self.max_length - x.size(-1)), value=self.pad_value - ) - elif pad_to > 0: - pad_amt = x.size(-1) % pad_to - if pad_amt != 0: - x = nn.functional.pad(x, (0, pad_to - pad_amt), value=self.pad_value) - - return x, seq_len - - -class CohereASRFeatureExtractor(SequenceFeatureExtractor): - """HF-compatible feature extractor wrapping FilterbankFeatures.""" - - model_input_names = ["input_features"] - - def __init__( - self, - feature_size=64, - sampling_rate=16000, - padding_value=0.0, - max_duration=30, - n_window_size=320, - n_window_stride=160, - window="hann", - normalize="per_feature", - n_fft=None, - preemph=0.97, - lowfreq=0, - highfreq=None, - log=True, - log_zero_guard_type="add", - log_zero_guard_value=2**-24, - dither=CONSTANT, - pad_to=16, - frame_splicing=1, - exact_pad=False, - mag_power=2.0, - nb_augmentation_prob=0.0, - nb_max_freq=4000, - mel_norm="slaney", - stft_exact_pad=False, - stft_conv=False, - device="cpu", - **kwargs, - ): - super().__init__( - feature_size=feature_size, - sampling_rate=sampling_rate, - padding_value=padding_value, - **kwargs, - ) - self.max_duration = max_duration - self.hop_length = n_window_stride - self._device = torch.device(device) - self._fb_config = dict( - sample_rate=sampling_rate, - n_window_size=n_window_size, - n_window_stride=n_window_stride, - window=window, - normalize=normalize, - n_fft=n_fft, - preemph=preemph, - nfilt=feature_size, - lowfreq=lowfreq, - highfreq=highfreq, - log=log, - log_zero_guard_type=log_zero_guard_type, - log_zero_guard_value=log_zero_guard_value, - dither=dither, - pad_to=pad_to, - max_duration=max_duration, - frame_splicing=frame_splicing, - exact_pad=exact_pad, - pad_value=padding_value, - mag_power=mag_power, - nb_augmentation_prob=nb_augmentation_prob, - nb_max_freq=nb_max_freq, - mel_norm=mel_norm, - stft_exact_pad=stft_exact_pad, - stft_conv=stft_conv, - device=device, - ) - self._filterbank: FilterbankFeatures | None = None - - @property - def filterbank(self) -> FilterbankFeatures: - if self._filterbank is None: - fb = FilterbankFeatures(**self._fb_config) - fb.eval() - self._filterbank = fb.to(self._device) - return self._filterbank - - def get_seq_len(self, seq_len): - return self.filterbank.get_seq_len(seq_len) - - def __call__( - self, - raw_speech, - sampling_rate=None, - return_tensors=None, - **kwargs, - ) -> BatchFeature: - if isinstance(raw_speech, np.ndarray): - raw_speech = [raw_speech] - - seq_len = torch.tensor([s.shape[0] for s in raw_speech]) - - max_len = max(s.shape[0] for s in raw_speech) - padded = np.zeros((len(raw_speech), max_len), dtype=np.float32) - for i, s in enumerate(raw_speech): - padded[i, : s.shape[0]] = s - - audio_tensor = torch.from_numpy(padded).to(self._device) - seq_len = seq_len.to(self._device) - - with torch.no_grad(): - input_features, length = self.filterbank(audio_tensor, seq_len) - - result = BatchFeature( - {"input_features": input_features.cpu(), "length": length.cpu()} - ) - if return_tensors is not None: - result = result.convert_to_tensors(return_tensors) - return result - - -class CohereASRProcessor(ProcessorMixin): - """HF-compatible processor combining CohereASRFeatureExtractor and a - tokenizer.""" - - feature_extractor_class = "CohereASRFeatureExtractor" - tokenizer_class = "AutoTokenizer" - - def __init__(self, feature_extractor, tokenizer): - super().__init__(feature_extractor, tokenizer) - - def __call__( - self, - text=None, - audio=None, - sampling_rate=None, - return_tensors=None, - **kwargs, - ): - if audio is not None: - result = self.feature_extractor( - audio, - sampling_rate=sampling_rate, - return_tensors=return_tensors, - ) - else: - result = BatchFeature() - - if text is not None: - text_inputs = self.tokenizer(text, return_tensors=return_tensors, **kwargs) - result["input_ids"] = text_inputs["input_ids"] - - return result - - -AutoFeatureExtractor.register("CohereASRFeatureExtractor", CohereASRFeatureExtractor) -AutoProcessor.register("CohereASRProcessor", CohereASRProcessor) diff --git a/vllm/transformers_utils/processors/deepseek_vl2.py b/vllm/transformers_utils/processors/deepseek_vl2.py deleted file mode 100644 index 6c9966abf8..0000000000 --- a/vllm/transformers_utils/processors/deepseek_vl2.py +++ /dev/null @@ -1,404 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -# ruff: noqa: E501 -# coding=utf-8 -# adapted from https://github.com/deepseek-ai/DeepSeek-VL2/blob/ff23960c5cf9e6874b44be38af930cfb0ccbb620/deepseek_vl2/models/processing_deepseek_vl_v2.py -# Copyright (c) 2023-2024 DeepSeek. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy of -# this software and associated documentation files (the "Software"), to deal in -# the Software without restriction, including without limitation the rights to -# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of -# the Software, and to permit persons to whom the Software is furnished to do so, -# subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS -# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR -# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER -# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -import math -from typing import Any - -import torch -import torchvision.transforms as T -from PIL import Image, ImageOps -from transformers import BatchFeature, LlamaTokenizerFast -from transformers.processing_utils import ProcessorMixin - - -class ImageTransform: - def __init__( - self, - mean: tuple[float, float, float] = (0.5, 0.5, 0.5), - std: tuple[float, float, float] = (0.5, 0.5, 0.5), - normalize: bool = True, - ): - self.mean = mean - self.std = std - self.normalize = normalize - - transform_pipelines = [T.ToTensor()] - - if normalize: - transform_pipelines.append(T.Normalize(mean, std)) - - self.transform = T.Compose(transform_pipelines) - - def __call__(self, pil_img: Image.Image): - x = self.transform(pil_img) - return x - - -class DeepseekVLV2Processor(ProcessorMixin): - tokenizer_class = ("LlamaTokenizer", "LlamaTokenizerFast") - attributes = ["tokenizer"] - - def __init__( - self, - tokenizer: LlamaTokenizerFast, - candidate_resolutions: tuple[tuple[int, int]], - patch_size: int, - downsample_ratio: int, - image_mean: tuple[float, float, float] = (0.5, 0.5, 0.5), - image_std: tuple[float, float, float] = (0.5, 0.5, 0.5), - normalize: bool = True, - image_token: str = "", - pad_token: str = "<|▁pad▁|>", - add_special_token: bool = False, - sft_format: str = "deepseek", - mask_prompt: bool = True, - ignore_id: int = -100, - **kwargs, - ): - self.candidate_resolutions = candidate_resolutions - self.image_size = candidate_resolutions[0][0] - self.patch_size = patch_size - self.image_mean = image_mean - self.image_std = image_std - self.normalize = normalize - self.downsample_ratio = downsample_ratio - - self.image_transform = ImageTransform( - mean=image_mean, std=image_std, normalize=normalize - ) - self.tokenizer = tokenizer - self.tokenizer.padding_side = "left" # must set this,padding side with make a difference in batch inference - - # add the pad_token as special token to use 'tokenizer.pad_token' and 'tokenizer.pad_token_id' - if tokenizer.pad_token is None: - self.tokenizer.add_special_tokens({"pad_token": pad_token}) - - # add image token - image_token_id = self.tokenizer.vocab.get(image_token) - if image_token_id is None: - special_tokens = [image_token] - special_tokens_dict = {"additional_special_tokens": special_tokens} - self.tokenizer.add_special_tokens(special_tokens_dict) - self.image_token_id = self.tokenizer.vocab.get(image_token) - - # add five special tokens for grounding-related tasks - # <|ref|>, <|/ref|>, <|det|>, <|/det|>, <|grounding|> - special_tokens = ["<|ref|>", "<|/ref|>", "<|det|>", "<|/det|>", "<|grounding|>"] - special_tokens_dict = {"additional_special_tokens": special_tokens} - self.tokenizer.add_special_tokens(special_tokens_dict) - - # add special tokens for SFT data - special_tokens = ["<|User|>", "<|Assistant|>"] - special_tokens_dict = {"additional_special_tokens": special_tokens} - self.tokenizer.add_special_tokens(special_tokens_dict) - - self.image_token = image_token - self.pad_token = pad_token - self.add_special_token = add_special_token - self.sft_format = sft_format - self.mask_prompt = mask_prompt - self.ignore_id = ignore_id - - super().__init__( - tokenizer, - **kwargs, - ) - - def select_best_resolution(self, image_size): - # used for cropping - original_width, original_height = image_size - best_fit = None - max_effective_resolution = 0 - min_wasted_resolution = float("inf") - - for width, height in self.candidate_resolutions: - scale = min(width / original_width, height / original_height) - downscaled_width, downscaled_height = ( - int(original_width * scale), - int(original_height * scale), - ) - effective_resolution = min( - downscaled_width * downscaled_height, original_width * original_height - ) - wasted_resolution = (width * height) - effective_resolution - - if effective_resolution > max_effective_resolution or ( - effective_resolution == max_effective_resolution - and wasted_resolution < min_wasted_resolution - ): - max_effective_resolution = effective_resolution - min_wasted_resolution = wasted_resolution - best_fit = (width, height) - - return best_fit - - @property - def bos_id(self): - return self.tokenizer.bos_token_id - - @property - def eos_id(self): - return self.tokenizer.eos_token_id - - @property - def pad_id(self): - return self.tokenizer.pad_token_id - - def encode(self, text: str, bos: bool = True, eos: bool = False): - t = self.tokenizer.encode(text, add_special_tokens=False) - - if bos: - t = [self.bos_id] + t - if eos: - t = t + [self.eos_id] - - return t - - def decode(self, t: list[int], **kwargs) -> str: - return self.tokenizer.decode(t, **kwargs) - - def process_one( - self, - prompt: str, - images: list[Image.Image], - inference_mode: bool = True, - **kwargs: Any, - ): - """Args: - prompt (str): the formatted prompt; - images (list[ImageType]): the list of images; - inference_mode (bool): if True, then remove the last eos token; - **kwargs: Additional keyword arguments. - - Returns: - outputs (BaseProcessorOutput): the output of the processor, - - input_ids (torch.LongTensor): [N + image tokens] - - target_ids (torch.LongTensor): [N + image tokens] - - pixel_values (torch.FloatTensor): [n_patches, 3, H, W] - - image_id (int): the id of the image token - - num_image_tokens (list[int]): the number of image tokens - - """ - if prompt is None or images is None: - raise ValueError("prompt and images must be used at the same time.") - - sft_format = prompt - ( - tokenized_str, - images_list, - images_seq_mask, - images_spatial_crop, - num_image_tokens, - ) = self.tokenize_with_images( - sft_format, images, bos=True, eos=True, cropping=len(images) <= 2 - ) - masked_tokenized_str = [] - for token_index in tokenized_str: - if token_index != self.image_token_id: - masked_tokenized_str.append(token_index) - else: - masked_tokenized_str.append(self.ignore_id) - - assert ( - len(tokenized_str) == len(images_seq_mask) == len(masked_tokenized_str) - ), ( - f"tokenized_str's length {len(tokenized_str)}, input_ids' length {len(masked_tokenized_str)}, " - f"imags_seq_mask's length {len(images_seq_mask)}, are not equal" - ) - - input_ids = torch.LongTensor(tokenized_str) - target_ids = torch.LongTensor(masked_tokenized_str) - images_seq_mask = torch.tensor(images_seq_mask, dtype=torch.bool) - - # set input_ids < 0 | input_ids == self.image_token_id as ignore_id - target_ids[(input_ids < 0) | (input_ids == self.image_token_id)] = ( - self.ignore_id - ) - input_ids[input_ids < 0] = self.pad_id - - if inference_mode: - # Remove the ending eos token - assert input_ids[-1] == self.eos_id - input_ids = input_ids[:-1] - target_ids = target_ids[:-1] - images_seq_mask = images_seq_mask[:-1] - - if len(images_list) == 0: - pixel_values = torch.zeros((1, 3, self.image_size, self.image_size)) - images_spatial_crop = torch.zeros((1, 2), dtype=torch.long) - else: - pixel_values = torch.stack(images_list, dim=0) - images_spatial_crop = torch.tensor(images_spatial_crop, dtype=torch.long) - - input_ids = input_ids.unsqueeze(0) - - prepare = BatchFeature( - data=dict( - input_ids=input_ids, - pixel_values=pixel_values, - images_seq_mask=images_seq_mask, - images_spatial_crop=images_spatial_crop, - num_image_tokens=num_image_tokens, - ), - tensor_type="pt", - ) - return prepare - - def __call__( - self, - *, - text: str, - images: list[Image.Image], - inference_mode: bool = True, - **kwargs: Any, - ): - """Args: - text (str): the formatted prompt; - images (list[ImageType]): the list of images; - inference_mode (bool): if True, then remove the last eos token; - **kwargs: - - Returns: - outputs (BaseProcessorOutput): the output of the processor, - - input_ids (torch.LongTensor): [N + image tokens] - - images (torch.FloatTensor): [n_images, 3, H, W] - - image_id (int): the id of the image token - - num_image_tokens (list[int]): the number of image tokens - - """ - prepare = self.process_one( - prompt=text, - images=images, - inference_mode=inference_mode, - ) - - return prepare - - def tokenize_with_images( - self, - conversation: str, - images: list[Image.Image], - bos: bool = True, - eos: bool = True, - cropping: bool = True, - ): - """Tokenize text with tags.""" - num_image_tags = conversation.count(self.image_token) - if num_image_tags != len(images): - raise ValueError( - f"Number of {self.image_token!r} tokens in prompt " - f"({num_image_tags}) does not match number of images " - f"({len(images)})." - ) - text_splits = conversation.split(self.image_token) - images_list, images_seq_mask, images_spatial_crop = [], [], [] - num_image_tokens = [] - tokenized_str = [] - for text_sep, image in zip(text_splits, images): - """encode text_sep""" - tokenized_sep = self.encode(text_sep, bos=False, eos=False) - tokenized_str += tokenized_sep - images_seq_mask += [False] * len(tokenized_sep) - - """select best resolution for anyres""" - if cropping: - best_width, best_height = self.select_best_resolution(image.size) - else: - best_width, best_height = self.image_size, self.image_size - - """process the global view""" - global_view = ImageOps.pad( - image, - (self.image_size, self.image_size), - color=tuple(int(x * 255) for x in self.image_transform.mean), - ) - images_list.append(self.image_transform(global_view)) - - """process the local views""" - local_view = ImageOps.pad( - image, - (best_width, best_height), - color=tuple(int(x * 255) for x in self.image_transform.mean), - ) - for i in range(0, best_height, self.image_size): - for j in range(0, best_width, self.image_size): - images_list.append( - self.image_transform( - local_view.crop( - (j, i, j + self.image_size, i + self.image_size) - ) - ) - ) - - """record height / width crop num""" - num_width_tiles, num_height_tiles = ( - best_width // self.image_size, - best_height // self.image_size, - ) - images_spatial_crop.append([num_width_tiles, num_height_tiles]) - - """add image tokens""" - h = w = math.ceil( - (self.image_size // self.patch_size) / self.downsample_ratio - ) - # global views tokens h * (w + 1), 1 is for line separator - tokenized_image = [self.image_token_id] * h * (w + 1) - # add a separator between global and local views - tokenized_image += [self.image_token_id] - # local views tokens, (num_height_tiles * h) * (num_width_tiles * w + 1) - tokenized_image += ( - [self.image_token_id] - * (num_height_tiles * h) - * (num_width_tiles * w + 1) - ) - - tokenized_str += tokenized_image - images_seq_mask += [True] * len(tokenized_image) - num_image_tokens.append(len(tokenized_image)) - - """process the last text split""" - tokenized_sep = self.encode(text_splits[-1], bos=False, eos=False) - tokenized_str += tokenized_sep - images_seq_mask += [False] * len(tokenized_sep) - - """add the bos and eos tokens""" - if bos: - tokenized_str = [self.bos_id] + tokenized_str - images_seq_mask = [False] + images_seq_mask - if eos: - tokenized_str = tokenized_str + [self.eos_id] - images_seq_mask = images_seq_mask + [False] - - assert len(tokenized_str) == len(images_seq_mask), ( - f"tokenize_with_images func: tokenized_str's length {len(tokenized_str)} is not equal to imags_seq_mask's length {len(images_seq_mask)}" - ) - - return ( - tokenized_str, - images_list, - images_seq_mask, - images_spatial_crop, - num_image_tokens, - ) diff --git a/vllm/transformers_utils/processors/fireredasr2.py b/vllm/transformers_utils/processors/fireredasr2.py deleted file mode 100644 index 6225e88e6a..0000000000 --- a/vllm/transformers_utils/processors/fireredasr2.py +++ /dev/null @@ -1,345 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from typing import TYPE_CHECKING - -import numpy as np -import torch -import torch.nn.functional as F -from transformers import ( - AutoFeatureExtractor, - BatchFeature, -) -from transformers.feature_extraction_sequence_utils import SequenceFeatureExtractor -from transformers.processing_utils import ProcessorMixin -from transformers.utils import TensorType - -from vllm.logger import init_logger -from vllm.utils.import_utils import LazyLoader - -if TYPE_CHECKING: - import kaldi_native_fbank as knf -else: - knf = LazyLoader("knf", globals(), "kaldi_native_fbank") - - -logger = init_logger(__name__) - - -class CMVN: - def __init__(self, dim, means, inverse_std_variences): - self.dim, self.means, self.inverse_std_variences = ( - dim, - np.array(means), - np.array(inverse_std_variences), - ) - - def __call__(self, x): - assert x.shape[-1] == self.dim, "CMVN dim mismatch" - out = x - self.means - out = out * self.inverse_std_variences - return out - - -class KaldifeatFbank: - def __init__(self, num_mel_bins=80, frame_length=25, frame_shift=10, dither=1.0): - self.dither = dither - opts = knf.FbankOptions() - opts.frame_opts.dither = dither - opts.mel_opts.num_bins = num_mel_bins - opts.frame_opts.snip_edges = True - opts.mel_opts.debug_mel = False - self.opts = opts - - def __call__(self, sample_rate, wav_np, is_train=False): - dither = self.dither if is_train else 0.0 - self.opts.frame_opts.dither = dither - fbank = knf.OnlineFbank(self.opts) - - fbank.accept_waveform(sample_rate, wav_np.tolist()) - feat = [] - for i in range(fbank.num_frames_ready): - feat.append(fbank.get_frame(i)) - if len(feat) == 0: - print("Check data, len(feat) == 0", wav_np, flush=True) - return np.zeros((0, self.opts.mel_opts.num_bins)) - feat = np.vstack(feat) - return feat - - -class FireRedASR2FeatureExtractor(SequenceFeatureExtractor): - r"""Constructs a FireRedASR2 feature extractor. - - This feature extractor inherits from [`~feature_extraction_sequence_ - utils.SequenceFeatureExtractor`] which contains most of the main - methods. Users should refer to this superclass for more information - regarding those methods. - - This class extracts mel-filter bank features from raw speech using a custom - numpy implementation of the `Short Time Fourier Transform` which should - match pytorch's `torch.stft` equivalent. - - Args: - feature_size (`int`, *optional*, defaults to 80): - The feature dimension of the extracted features. - sampling_rate (`int`, *optional*, defaults to 16000): - The sampling rate at which the audio files should be digitalized - expressed in hertz (Hz). - chunk_length (`int`, *optional*, defaults to 30): - The maximum number of chunks of `sampling_rate` samples used to - trim and pad longer or shorter audio sequences. - padding_value (`float`, *optional*, defaults to 0.0): - Padding value used to pad the audio. Should correspond to silences. - dither (`float`, *optional*, defaults to 0.0): - Adds dithering. In other words, adds a small Gaussian noise to each frame. - E.g. use 0.0001 to add dithering with a normal distribution centered - around 0.0 with standard deviation 0.0001 (assuming [-1,+1] range - of raw_speech). The value 0.0 means no dithering. - Dithering has similar effect as `spectrogram(mel_floor=...)`. It reduces - the high log_mel_fbank values for signals with hard-zero sections, - when VAD cutoff is present in the signal. - - """ - - model_input_names = ["input_features"] - - def __init__( - self, - feature_size=80, - sampling_rate=16000, - chunk_length=30, - padding_value=0.0, - return_attention_mask=False, - dim=80, - means=None, - inverse_std_variences=None, - num_mel_bins=80, - frame_length=25, - frame_shift=10, - dither=0.0, - max_length=3000, - downsample_rate=2, - left_context=3, - right_context=3, - **kwargs, - ): - super().__init__( - feature_size=feature_size, - sampling_rate=sampling_rate, - padding_value=padding_value, - return_attention_mask=return_attention_mask, - **kwargs, - ) - self.chunk_length = chunk_length - self.max_length = max_length - self.dim = dim - self.means = means - self.inverse_std_variences = inverse_std_variences - self.num_mel_bins = num_mel_bins - self.frame_length = frame_length - self.frame_shift = frame_shift - self.dither = dither - self.sampling_rate = sampling_rate - self.downsample_rate = downsample_rate - self.context = left_context + 1 + right_context - - def __call__( - self, - raw_speech: np.ndarray | list[float] | list[np.ndarray] | list[list[float]], - truncation: bool = True, - pad_to_multiple_of: int | None = None, - return_tensors: str | TensorType | None = None, - return_attention_mask: bool | None = None, - padding: str | None = "max_length", - max_length: int | None = None, - sampling_rate: int | None = None, - do_normalize: bool | None = None, - **kwargs, - ) -> BatchFeature: - if sampling_rate != self.sampling_rate: - raise ValueError( - f"The model corresponding to this feature extractor: " - f"{self.__class__.__name__} was trained using a sampling " - f"rate of {self.sampling_rate}. Please make sure that the " - f"provided `raw_speech` input was sampled with " - f"{self.sampling_rate} and not {sampling_rate}." - ) - - def padding_position_is_0(padded_input, input_lengths): - N, T = padded_input.size()[:2] - mask = torch.ones((N, T)).to(padded_input.device) - for i in range(N): - mask[i, input_lengths[i] :] = 0 - mask = mask.unsqueeze(dim=1) - return mask.to(torch.uint8) - - # initialize the CMVN and Fbank objects - self.cmvn = CMVN(self.dim, self.means, self.inverse_std_variences) - self.fbank = KaldifeatFbank( - num_mel_bins=self.num_mel_bins, - frame_length=self.frame_length, - frame_shift=self.frame_shift, - dither=self.dither, - ) - - feats = [] - speech_lengths = [] - fake_token_lengths = [] - for speech in raw_speech: - """ - We must multiply by 32768 here because FireRedASR2 loads audio data - using kaldiio.load_mat, while vLLM loads audio data using pyav. - """ - speech = speech * 32768 - fbank = self.fbank(sampling_rate, speech) - fbank = self.cmvn(fbank) - fbank = torch.from_numpy(fbank).float() - length = fbank.size(0) - feats.append(fbank) - speech_lengths.append(length) - padded_input2 = fbank - padded_input2 = F.pad( - padded_input2, (0, 0, 0, self.context - 1), "constant", 0.0 - ) - src_mask = padding_position_is_0( - padded_input2[None, :, :], torch.tensor([length], dtype=torch.int32) - ) - x_mask = src_mask - mask = x_mask[:, :, :-2:2][:, :, :-2:2] - input_lengths = mask[:, -1, :].sum(dim=-1) - input_lengths = input_lengths // self.downsample_rate - fake_token_len = torch.clamp(input_lengths, min=1) - fake_token_lengths.append(fake_token_len) - - feats = torch.stack(feats, dim=0) - batched_speech = self.pad( - BatchFeature({"input_features": feats}), - padding=padding, - max_length=max_length if max_length else self.max_length, - truncation=truncation, - pad_to_multiple_of=pad_to_multiple_of, - return_attention_mask=return_attention_mask or do_normalize, - ) - - if return_tensors is not None: - batched_speech = batched_speech.convert_to_tensors(return_tensors) - - batched_speech["speech_lengths"] = torch.tensor(speech_lengths) - batched_speech["fake_token_lengths"] = torch.concat(fake_token_lengths) - return batched_speech - - -class FireRedASR2Processor(ProcessorMixin): - r"""Constructs a FireRedASR2 processor which wraps a FireRedASR2 feature extractor - and a FireRedASR2 tokenizer into a single processor. - - [`FireRedASR2Processor`] offers all the functionalities of - [`FireRedASR2FeatureExtractor`] and [`Qwen2Tokenizer`]. See the - [`~FireRedASR2Processor.__call__`] and [`~FireRedASR2Processor.decode`] for more - information. - - Args: - feature_extractor (`FireRedASR2FeatureExtractor`): An instance of - [`FireRedASR2FeatureExtractor`]. - The feature extractor is a required input. - tokenizer (`Qwen2Tokenizer`): - An instance of [`Qwen2Tokenizer`]. The tokenizer is a required - input. - - """ - - feature_extractor_class = "FireRedASR2FeatureExtractor" - tokenizer_class = ("Qwen2Tokenizer", "Qwen2TokenizerFast") - - def __init__( - self, - feature_extractor, - tokenizer, - audio_token="<|AUDIO|>", - ): - super().__init__(feature_extractor, tokenizer) - self.current_processor = self.feature_extractor - self._in_target_context_manager = False - self.audio_token = ( - tokenizer.audio_token if hasattr(tokenizer, "audio_token") else audio_token - ) - self.audio_token_id = tokenizer.convert_tokens_to_ids(self.audio_token) - - def get_decoder_prompt_ids(self, task=None, language=None, no_timestamps=True): - return self.tokenizer.get_decoder_prompt_ids( - task=task, language=language, no_timestamps=no_timestamps - ) - - def __call__(self, *args, **kwargs): - """Forwards the `audio` argument to FireRedASR2FeatureExtractor's - [`~FireRedASR2FeatureExtractor.__call__`] and the `text` argument to - [`~Qwen2Tokenizer.__call__`]. Please refer to the docstring of the - above two methods for more information. - """ - if self._in_target_context_manager: - return self.current_processor(*args, **kwargs) - - audio = kwargs.pop("audio", None) - sampling_rate = kwargs.pop("sampling_rate", None) - text = kwargs.pop("text", None) - if len(args) > 0: - audio = args[0] - args = args[1:] - - if text is None: - raise ValueError("You need to specify `text` input to process.") - elif isinstance(text, str): - text = [text] - elif not isinstance(text, list) and not isinstance(text[0], str): - raise ValueError( - "Invalid input text. Please provide a string, or a list of strings" - ) - - if audio is not None: - # ensure we have as much audios as audio tokens - num_audio_tokens = sum(sample.count(self.audio_token) for sample in text) - num_audios = 1 if type(audio) is np.ndarray else len(audio) - if num_audio_tokens != num_audios: - raise ValueError( - f"Found {num_audio_tokens} {self.audio_token} token{'s' if num_audio_tokens > 1 else ''} in provided text but received {num_audios} audio{'s' if num_audios > 1 else ''}" # noqa: E501 - ) - inputs = self.feature_extractor( - audio, *args, sampling_rate=sampling_rate, **kwargs - ) - - expanded_text = [] - for sample in text: - replace_str = [] - while self.audio_token in sample: - num_audio_tokens = int(inputs["fake_token_lengths"].item()) - - expanded_audio_token = self.audio_token * num_audio_tokens - - replace_str.append(expanded_audio_token) - sample = sample.replace(self.audio_token, "", 1) - - while "" in sample: - sample = sample.replace("", replace_str.pop(0), 1) - expanded_text.append(sample) - text = expanded_text - - if text is not None: - encodings = self.tokenizer(text, **kwargs) - - if text is None: - return inputs - - elif audio is None: - return encodings - else: - inputs["labels"] = encodings["input_ids"] - - return inputs - - def get_prompt_ids(self, text: str, return_tensors="np"): - return self.tokenizer.get_prompt_ids(text, return_tensors=return_tensors) - - -AutoFeatureExtractor.register( - "FireRedASR2FeatureExtractor", FireRedASR2FeatureExtractor -) diff --git a/vllm/transformers_utils/processors/funasr.py b/vllm/transformers_utils/processors/funasr.py deleted file mode 100644 index 0a2f9ca6e1..0000000000 --- a/vllm/transformers_utils/processors/funasr.py +++ /dev/null @@ -1,429 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - - -import numpy as np -import torch -import torch.nn as nn -import torchaudio.compliance.kaldi as kaldi -from torch.nn.utils.rnn import pad_sequence -from transformers import ( - AutoFeatureExtractor, - BatchFeature, -) -from transformers.feature_extraction_sequence_utils import SequenceFeatureExtractor -from transformers.processing_utils import ProcessorMixin -from transformers.utils import TensorType - -from vllm.logger import init_logger - -logger = init_logger(__name__) - - -def apply_cmvn(inputs, cmvn): # noqa - """Apply CMVN with mvn data.""" - device = inputs.device - # dtype = inputs.dtype - frame, dim = inputs.shape - - means = cmvn[0:1, :dim] - vars = cmvn[1:2, :dim] - inputs += means.to(device) - inputs *= vars.to(device) - - return inputs.type(torch.float32) - - -def apply_lfr(inputs, lfr_m, lfr_n): - # LFR_inputs = [] - T = inputs.shape[0] - T_lfr = int(np.ceil(T / lfr_n)) - left_padding = inputs[0].repeat((lfr_m - 1) // 2, 1) - inputs = torch.vstack((left_padding, inputs)) - T = T + (lfr_m - 1) // 2 - feat_dim = inputs.shape[-1] - strides = (lfr_n * feat_dim, 1) - sizes = (T_lfr, lfr_m * feat_dim) - last_idx = (T - lfr_m) // lfr_n + 1 - num_padding = lfr_m - (T - last_idx * lfr_n) - if num_padding > 0: - num_padding = ( - (2 * lfr_m - 2 * T + (T_lfr - 1 + last_idx) * lfr_n) - / 2 - * (T_lfr - last_idx) - ) - inputs = torch.vstack([inputs] + [inputs[-1:]] * int(num_padding)) - LFR_outputs = inputs.as_strided(sizes, strides) - return LFR_outputs.clone().type(torch.float32) - - -def load_cmvn(cmvn_file): - with open(cmvn_file, encoding="utf-8") as f: - lines = f.readlines() - means_list = [] - vars_list = [] - for i in range(len(lines)): - line_item = lines[i].split() - if line_item[0] == "": - line_item = lines[i + 1].split() - if line_item[0] == "": - add_shift_line = line_item[3 : (len(line_item) - 1)] - means_list = list(add_shift_line) - continue - elif line_item[0] == "": - line_item = lines[i + 1].split() - if line_item[0] == "": - rescale_line = line_item[3 : (len(line_item) - 1)] - vars_list = list(rescale_line) - continue - means = np.array(means_list).astype(np.float32) - vars = np.array(vars_list).astype(np.float32) - cmvn = np.array([means, vars]) - cmvn = torch.as_tensor(cmvn, dtype=torch.float32) - return cmvn - - -class WavFrontend(nn.Module): - """Conventional frontend structure for ASR.""" - - def __init__( - self, - cmvn_file: str = "null", - fs: int = 16000, - window: str = "hamming", - n_mels: int = 80, - frame_length: int = 25, - frame_shift: int = 10, - filter_length_min: int = -1, - filter_length_max: int = -1, - lfr_m: int = 1, - lfr_n: int = 1, - dither: float = 1.0, - snip_edges: bool = True, - upsacle_samples: bool = True, - **kwargs, - ): - super().__init__() - self.fs = fs - self.window = window - self.n_mels = n_mels - self.frame_length = frame_length - self.frame_shift = frame_shift - self.filter_length_min = filter_length_min - self.filter_length_max = filter_length_max - self.lfr_m = lfr_m - self.lfr_n = lfr_n - self.cmvn_file = cmvn_file - self.dither = dither - self.snip_edges = snip_edges - self.upsacle_samples = upsacle_samples - self.cmvn = None if self.cmvn_file is None else load_cmvn(self.cmvn_file) - - def output_size(self) -> int: - return self.n_mels * self.lfr_m - - def forward( - self, - input: torch.Tensor, - input_lengths, - **kwargs, - ) -> tuple[torch.Tensor, torch.Tensor]: - batch_size = input.size(0) - feats = [] - feats_lens = [] - for i in range(batch_size): - waveform_length = input_lengths[i] - waveform = input[i][:waveform_length] - if self.upsacle_samples: - waveform = waveform * (1 << 15) - waveform = waveform.unsqueeze(0) - mat = kaldi.fbank( - waveform, - num_mel_bins=self.n_mels, - frame_length=min(self.frame_length, waveform_length / self.fs * 1000), - frame_shift=self.frame_shift, - dither=self.dither, - energy_floor=0.0, - window_type=self.window, - sample_frequency=self.fs, - snip_edges=self.snip_edges, - ) - - if self.lfr_m != 1 or self.lfr_n != 1: - mat = apply_lfr(mat, self.lfr_m, self.lfr_n) - if self.cmvn is not None: - mat = apply_cmvn(mat, self.cmvn) - feat_length = mat.size(0) - feats.append(mat) - feats_lens.append(feat_length) - - feats_lens = torch.as_tensor(feats_lens) - if batch_size == 1: - feats_pad = feats[0][None, :, :] - else: - feats_pad = pad_sequence(feats, batch_first=True, padding_value=0.0) - return feats_pad, feats_lens - - -class FunASRFeatureExtractor(SequenceFeatureExtractor): - r"""Constructs a FunASR feature extractor. - - This feature extractor inherits from [`~feature_extraction_sequence_ - utils.SequenceFeatureExtractor`] which contains most of the main - methods. Users should refer to this superclass for more information - regarding those methods. - - This class extracts mel-filter bank features from raw speech using a custom - numpy implementation of the `Short Time Fourier Transform` which should - match pytorch's `torch.stft` equivalent. - - Args: - feature_size (`int`, *optional*, defaults to 80): - The feature dimension of the extracted features. - sampling_rate (`int`, *optional*, defaults to 16000): - The sampling rate at which the audio files should be digitalized - expressed in hertz (Hz). - hop_length (`int`, *optional*, defaults to 160): - Length of the overlapping windows for the STFT used to obtain the - Mel Frequency coefficients. - chunk_length (`int`, *optional*, defaults to 30): - The maximum number of chunks of `sampling_rate` samples used to - trim and pad longer or shorter audio sequences. - n_fft (`int`, *optional*, defaults to 400): - Size of the Fourier transform. - padding_value (`float`, *optional*, defaults to 0.0): - Padding value used to pad the audio. Should correspond to silences. - dither (`float`, *optional*, defaults to 0.0): - Adds dithering. In other words, adds a small Gaussian noise to each frame. - E.g. use 0.0001 to add dithering with a normal distribution centered - around 0.0 with standard deviation 0.0001 (assuming [-1,+1] range - of raw_speech). The value 0.0 means no dithering. - Dithering has similar effect as `spectrogram(mel_floor=...)`. It reduces - the high log_mel_fbank values for signals with hard-zero sections, - when VAD cutoff is present in the signal. - - """ - - model_input_names = ["input_features"] - - def __init__( - self, - feature_size=80, - sampling_rate=16000, - hop_length=160, - chunk_length=30, - n_fft=400, - padding_value=0.0, - dither=0.0, - max_length=1000, - return_attention_mask=False, - **kwargs, - ): - super().__init__( - feature_size=feature_size, - sampling_rate=sampling_rate, - padding_value=padding_value, - return_attention_mask=return_attention_mask, - **kwargs, - ) - self.frontend_conf = kwargs.get("frontend_conf", {}) - self.max_length = max_length - self.n_fft = n_fft - self.hop_length = hop_length - self.chunk_length = chunk_length - self.n_samples = chunk_length * sampling_rate - self.nb_max_frames = self.n_samples // hop_length - self.sampling_rate = sampling_rate - self.dither = dither - - def extract_fbank( - self, data, data_len=None, data_type: str = "sound", frontend=None, **kwargs - ): - if isinstance(data, np.ndarray): - data = torch.from_numpy(data) - if len(data.shape) < 2: - data = data[None, :] # data: [batch, N] - data_len = [data.shape[1]] if data_len is None else data_len - elif isinstance(data, torch.Tensor): - if len(data.shape) < 2: - data = data[None, :] # data: [batch, N] - data_len = [data.shape[1]] if data_len is None else data_len - elif isinstance(data, (list, tuple)): - data_list, data_len = [], [] - for data_i in data: - if isinstance(data_i, np.ndarray): - data_i = torch.from_numpy(data_i) - data_list.append(data_i) - data_len.append(data_i.shape[0]) - data = pad_sequence(data_list, batch_first=True) - - data, data_len = frontend(data, data_len, **kwargs) - - if isinstance(data_len, (list, tuple)): - data_len = torch.tensor([data_len]) - return data.to(torch.float32), data_len.to(torch.int32) - - def __call__( - self, - raw_speech: np.ndarray | list[float] | list[np.ndarray] | list[list[float]], - truncation: bool = True, - pad_to_multiple_of: int | None = None, - return_tensors: str | TensorType | None = None, - return_attention_mask: bool | None = None, - padding: str | None = "max_length", - max_length: int | None = None, - sampling_rate: int | None = None, - do_normalize: bool | None = None, - device: str | None = "cpu", - return_token_timestamps: bool | None = None, - **kwargs, - ) -> BatchFeature: - frontend = WavFrontend(**self.frontend_conf, dither=self.dither) - - feats = [] - speech_lengths = [] - fake_token_lengths = [] - for speech in raw_speech: - feature, length = self.extract_fbank( - speech, - data_type=kwargs.get("data_type", "sound"), - frontend=frontend, - is_final=True, - ) - feats.append(feature) - speech_lengths.append(length) - olens = 1 + (length - 3 + 2 * 1) // 2 - olens = 1 + (olens - 3 + 2 * 1) // 2 - fake_token_len = (olens - 1) // 2 + 1 - fake_token_len = torch.clamp(fake_token_len, min=1) - fake_token_lengths.append(fake_token_len) - - feats = torch.concat(feats, dim=0) - batched_speech = self.pad( - BatchFeature({"input_features": feats}), - padding=padding, - max_length=max_length if max_length else self.max_length, - truncation=truncation, - pad_to_multiple_of=pad_to_multiple_of, - return_attention_mask=return_attention_mask or do_normalize, - ) - if return_tensors is not None: - batched_speech = batched_speech.convert_to_tensors(return_tensors) - - batched_speech["speech_lengths"] = torch.tensor(speech_lengths) - batched_speech["fake_token_lengths"] = torch.concat(fake_token_lengths) - return batched_speech - - -class FunASRProcessor(ProcessorMixin): - r"""Constructs a FunASR processor which wraps a FunASR feature extractor and - a FunASR tokenizer into a single processor. - - [`FunASRProcessor`] offers all the functionalities of - [`FunASRFeatureExtractor`] and [`Qwen2Tokenizer`]. See the - [`~FunASRProcessor.__call__`] and [`~FunASRProcessor.decode`] for more - information. - - Args: - feature_extractor (`FunASRFeatureExtractor`): An instance of - [`FunASRFeatureExtractor`]. - The feature extractor is a required input. - tokenizer (`Qwen2Tokenizer`): - An instance of [`Qwen2Tokenizer`]. The tokenizer is a required - input. - - """ - - feature_extractor_class = "FunASRFeatureExtractor" - tokenizer_class = ("Qwen2Tokenizer", "Qwen2TokenizerFast") - - def __init__( - self, - feature_extractor, - tokenizer, - audio_token="<|AUDIO|>", - ): - super().__init__(feature_extractor, tokenizer) - self.current_processor = self.feature_extractor - self._in_target_context_manager = False - self.audio_token = ( - tokenizer.audio_token if hasattr(tokenizer, "audio_token") else audio_token - ) - self.audio_token_id = tokenizer.convert_tokens_to_ids(self.audio_token) - - def get_decoder_prompt_ids(self, task=None, language=None, no_timestamps=True): - return self.tokenizer.get_decoder_prompt_ids( - task=task, language=language, no_timestamps=no_timestamps - ) - - def __call__(self, *args, **kwargs): - """Forwards the `audio` argument to FunASRFeatureExtractor's - [`~FunASRFeatureExtractor.__call__`] and the `text` argument to - [`~Qwen2Tokenizer.__call__`]. Please refer to the docstring of the - above two methods for more information. - """ - if self._in_target_context_manager: - return self.current_processor(*args, **kwargs) - - audio = kwargs.pop("audio", None) - sampling_rate = kwargs.pop("sampling_rate", None) - text = kwargs.pop("text", None) - if len(args) > 0: - audio = args[0] - args = args[1:] - - if text is None: - raise ValueError("You need to specify `text` input to process.") - elif isinstance(text, str): - text = [text] - elif not isinstance(text, list) and not isinstance(text[0], str): - raise ValueError( - "Invalid input text. Please provide a string, or a list of strings" - ) - - if audio is not None: - # ensure we have as much audios as audio tokens - num_audio_tokens = sum(sample.count(self.audio_token) for sample in text) - num_audios = 1 if type(audio) is np.ndarray else len(audio) - if num_audio_tokens != num_audios: - raise ValueError( - f"Found {num_audio_tokens} {self.audio_token} token{'s' if num_audio_tokens > 1 else ''} in provided text but received {num_audios} audio{'s' if num_audios > 1 else ''}" # noqa: E501 - ) - inputs = self.feature_extractor( - audio, *args, sampling_rate=sampling_rate, **kwargs - ) - - expanded_text = [] - for sample in text: - replace_str = [] - while self.audio_token in sample: - num_audio_tokens = inputs["fake_token_lengths"].item() - - expanded_audio_token = self.audio_token * num_audio_tokens - - replace_str.append(expanded_audio_token) - sample = sample.replace(self.audio_token, "", 1) - - while "" in sample: - sample = sample.replace("", replace_str.pop(0), 1) - expanded_text.append(sample) - text = expanded_text - - if text is not None: - encodings = self.tokenizer(text, **kwargs) - - if text is None: - return inputs - - elif audio is None: - return encodings - else: - inputs["labels"] = encodings["input_ids"] - - return inputs - - def get_prompt_ids(self, text: str, return_tensors="np"): - return self.tokenizer.get_prompt_ids(text, return_tensors=return_tensors) - - -AutoFeatureExtractor.register("FunASRFeatureExtractor", FunASRFeatureExtractor) diff --git a/vllm/transformers_utils/processors/glm4v.py b/vllm/transformers_utils/processors/glm4v.py deleted file mode 100644 index 605cd7e34a..0000000000 --- a/vllm/transformers_utils/processors/glm4v.py +++ /dev/null @@ -1,35 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -# Adapted from -# https://github.com/zai-org/CogAgent -from transformers import PythonBackend -from transformers.image_processing_backends import TorchvisionBackend -from transformers.image_utils import PILImageResampling -from transformers.processing_utils import ProcessorMixin - - -class GLM4VImageProcessorFast(TorchvisionBackend): - """Port of https://huggingface.co/zai-org/glm-4v-9b/blob/main/tokenization_chatglm.py#L177 - to HF Transformers. - """ - - resample = PILImageResampling.BICUBIC - image_mean = [0.48145466, 0.4578275, 0.40821073] - image_std = [0.26862954, 0.26130258, 0.27577711] - size = {"height": 1120, "width": 1120} - do_resize = True - do_rescale = True - do_normalize = True - - -class GLM4VProcessor(ProcessorMixin): - attributes = ["image_processor", "tokenizer"] - - def __init__( - self, - image_processor: GLM4VImageProcessorFast, - tokenizer: PythonBackend, - ) -> None: - self.image_processor = image_processor - self.tokenizer = tokenizer diff --git a/vllm/transformers_utils/processors/h2ovl.py b/vllm/transformers_utils/processors/h2ovl.py deleted file mode 100644 index e40d81cb16..0000000000 --- a/vllm/transformers_utils/processors/h2ovl.py +++ /dev/null @@ -1,387 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -# adapted from https://huggingface.co/h2oai/h2ovl-mississippi-2b/blob/main/modeling_h2ovl_chat.py -# https://huggingface.co/h2oai/h2ovl-mississippi-2b/blob/main/image_process.py -# -------------------------------------------------------- -# H2OVL-Mississippi -# Copyright (c) 2024 H2O.AI -# Licensed under Apache 2.0 License [see LICENSE for details] -# -------------------------------------------------------- -import torch -from PIL import Image - -from vllm.tokenizers.hf import HfTokenizer - -from .internvl import ( - InternVLImageProcessor, - InternVLProcessor, - build_transform, - find_closest_aspect_ratio, - get_internvl_target_ratios, -) - - -def resolve_h2ovl_min_max_num( - *, - min_dynamic_patch: int, - max_dynamic_patch: int, - dynamic_image_size: bool, - use_thumbnail: bool, -) -> tuple[int, int]: - min_dynamic_patch = min_dynamic_patch if dynamic_image_size else 1 - max_dynamic_patch = max_dynamic_patch if dynamic_image_size else 1 - - if use_thumbnail and max_dynamic_patch != 1: - max_dynamic_patch += 1 - - return min_dynamic_patch, max_dynamic_patch - - -def get_h2ovl_target_ratios( - min_num: int, - max_num: int, - *, - prior_aspect_ratio: tuple[int, int] | None, -) -> list[tuple[int, int]]: - target_ratios = get_internvl_target_ratios(min_num, max_num) - - # if prior_aspect_ratio is provided, filter the target ratios - if prior_aspect_ratio is not None: - target_ratios = [ - ratio - for ratio in target_ratios - if prior_aspect_ratio[0] % ratio[0] != 0 - and prior_aspect_ratio[1] % ratio[1] != 0 - ] - - return target_ratios - - -# modified to include blocks generated in second pass -def calculate_h2ovl_targets( - *, - orig_width: int, - orig_height: int, - target_ratios: list[tuple[int, int]], - image_size: int, - use_thumbnail: bool, -) -> tuple[int, int, int, tuple[int, int]]: - aspect_ratio = orig_width / orig_height - - # find the closest aspect ratio to the target - target_aspect_ratio = find_closest_aspect_ratio( - aspect_ratio, - target_ratios, - width=orig_width, - height=orig_height, - image_size=image_size, - ) - - # calculate the target width and height - target_width = image_size * target_aspect_ratio[0] - target_height = image_size * target_aspect_ratio[1] - blocks = target_aspect_ratio[0] * target_aspect_ratio[1] - - # add thumbnail image if num_blocks != 1 - if use_thumbnail and blocks != 1: - blocks += 1 - - return blocks, target_width, target_height, target_aspect_ratio - - -# adapted from https://huggingface.co/OpenGVLab/InternVL2-1B -# refactored to handle prior_aspect_ratio -def dynamic_preprocess_h2ovl( - image: Image.Image, - *, - target_ratios: list[tuple[int, int]], - image_size: int, - use_thumbnail: bool, -) -> tuple[list[Image.Image], tuple[int, int]]: - orig_width, orig_height = image.size - - # calculate the number of blocks without thumbnail - ( - blocks, - target_width, - target_height, - target_aspect_ratio, - ) = calculate_h2ovl_targets( - orig_width=orig_width, - orig_height=orig_height, - target_ratios=target_ratios, - image_size=image_size, - use_thumbnail=False, - ) - - # resize the image - resized_img = image.resize((target_width, target_height)) - processed_images = [] - for i in range(blocks): - box = ( - (i % (target_width // image_size)) * image_size, - (i // (target_width // image_size)) * image_size, - ((i % (target_width // image_size)) + 1) * image_size, - ((i // (target_width // image_size)) + 1) * image_size, - ) - # split the image - split_img = resized_img.crop(box) - processed_images.append(split_img) - - assert len(processed_images) == blocks - - if use_thumbnail and len(processed_images) != 1: - thumbnail_img = image.resize((image_size, image_size)) - processed_images.append(thumbnail_img) - - return processed_images, target_aspect_ratio - - -def _preprocess_image( - image: Image.Image, - *, - input_size: int, - min_num: int, - max_num: int, - use_thumbnail: bool, - prior_aspect_ratio: tuple[int, int] | None, -) -> tuple[torch.Tensor, tuple[int, int]]: - target_ratios = get_h2ovl_target_ratios( - min_num, - max_num, - prior_aspect_ratio=prior_aspect_ratio, - ) - - transform = build_transform(input_size=input_size) - images, target_aspect_ratio = dynamic_preprocess_h2ovl( - image, - image_size=input_size, - use_thumbnail=use_thumbnail, - target_ratios=target_ratios, - ) - - pixel_values = torch.stack([transform(image) for image in images]) - return pixel_values, target_aspect_ratio - - -# refactored to use the _preprocess_image function -def image_to_pixel_values_h2ovl( - image: Image.Image, - *, - input_size: int, - min_num: int, - max_num: int, - use_thumbnail: bool, - use_msac: bool, -) -> torch.Tensor: - # when MSAC is turned on, we need to process the image twice - if use_msac: - # first pass - pixel_values1, aspect_ratio1 = _preprocess_image( - image, - input_size=input_size, - min_num=1, - max_num=max_num, - use_thumbnail=True, - prior_aspect_ratio=None, - ) - # second pass - pixel_values2, _ = _preprocess_image( - image, - input_size=input_size, - min_num=3, - max_num=max_num, - use_thumbnail=True, - prior_aspect_ratio=aspect_ratio1, - ) - # combine pixel values - pixel_values = torch.cat( - [pixel_values2[:-1], pixel_values1[:-1], pixel_values2[-1:]], 0 - ) - - else: - pixel_values, _ = _preprocess_image( - image, - input_size=input_size, - min_num=min_num, - max_num=max_num, - use_thumbnail=use_thumbnail, - prior_aspect_ratio=None, - ) - - return pixel_values - - -class H2OVLImageProcessor(InternVLImageProcessor): - def __init__( - self, - image_size: int, - min_dynamic_patch: int, - max_dynamic_patch: int, - dynamic_image_size: bool, - use_thumbnail: bool, - use_msac: bool, - ) -> None: - super().__init__( - image_size=image_size, - min_dynamic_patch=min_dynamic_patch, - max_dynamic_patch=max_dynamic_patch, - dynamic_image_size=dynamic_image_size, - use_thumbnail=use_thumbnail, - ) - - self.use_msac = use_msac - - def resolve_min_max_num( - self, - *, - min_dynamic_patch: int | None = None, - max_dynamic_patch: int | None = None, - dynamic_image_size: bool | None = None, - use_thumbnail: bool | None = None, - ) -> tuple[int, int]: - if min_dynamic_patch is None: - min_dynamic_patch = self.min_dynamic_patch - if max_dynamic_patch is None: - max_dynamic_patch = self.max_dynamic_patch - if dynamic_image_size is None: - dynamic_image_size = self.dynamic_image_size - if use_thumbnail is None: - use_thumbnail = self.use_thumbnail - - return resolve_h2ovl_min_max_num( - min_dynamic_patch=min_dynamic_patch, - max_dynamic_patch=max_dynamic_patch, - dynamic_image_size=dynamic_image_size, - use_thumbnail=use_thumbnail, - ) - - def _images_to_pixel_values_lst( - self, - images: list[Image.Image], - min_dynamic_patch: int | None = None, - max_dynamic_patch: int | None = None, - dynamic_image_size: bool | None = None, - ) -> list[torch.Tensor]: - use_msac = self.use_msac if len(images) == 1 else False - - min_num, max_num = self.resolve_min_max_num( - min_dynamic_patch=min_dynamic_patch, - max_dynamic_patch=max_dynamic_patch, - dynamic_image_size=dynamic_image_size, - use_thumbnail=False, # Applied in image_to_pixel_values - ) - - return [ - image_to_pixel_values_h2ovl( - image, - input_size=self.image_size, - min_num=min_num, - max_num=max_num, - use_thumbnail=self.use_thumbnail, - use_msac=use_msac, - ) - for image in images - ] - - -class H2OVLProcessor(InternVLProcessor): - def __init__( - self, - image_processor: H2OVLImageProcessor, - tokenizer: HfTokenizer, - *, - image_seq_length: int, - start_image_token: str = "", - end_image_token: str = "", - ctx_image_token: str = "", - ) -> None: - super().__init__( - image_processor=image_processor, - tokenizer=tokenizer, - image_seq_length=image_seq_length, - start_image_token=start_image_token, - end_image_token=end_image_token, - ctx_image_token=ctx_image_token, - ) - - self.image_processor: H2OVLImageProcessor - - def resolve_target_ratios( - self, - *, - min_dynamic_patch: int | None = None, - max_dynamic_patch: int | None = None, - dynamic_image_size: bool | None = None, - use_thumbnail: bool | None = None, - prior_aspect_ratio: tuple[int, int] | None = None, - override_min_num: int | None = None, - ) -> list[tuple[int, int]]: - min_num, max_num = self.image_processor.resolve_min_max_num( - min_dynamic_patch=min_dynamic_patch, - max_dynamic_patch=max_dynamic_patch, - dynamic_image_size=dynamic_image_size, - use_thumbnail=use_thumbnail, - ) - if override_min_num is not None: - min_num = override_min_num - - return get_h2ovl_target_ratios( - min_num, - max_num, - prior_aspect_ratio=prior_aspect_ratio, - ) - - def get_num_image_tokens( - self, - *, - image_width: int, - image_height: int, - use_msac: bool | None = None, - ) -> int: - image_processor = self.image_processor - use_msac = image_processor.use_msac if use_msac is None else use_msac - - use_thumbnail = image_processor.use_thumbnail - - if use_msac: - target_ratios_1 = self.resolve_target_ratios( - use_thumbnail=False, # Applied in calculate_targets - override_min_num=1, - ) - num_patches_1, _, _, aspect_ratio_1 = calculate_h2ovl_targets( - orig_width=image_width, - orig_height=image_height, - image_size=image_processor.image_size, - target_ratios=target_ratios_1, - use_thumbnail=True, - ) - - target_ratios_2 = self.resolve_target_ratios( - use_thumbnail=False, # Applied in calculate_targets - prior_aspect_ratio=aspect_ratio_1, - override_min_num=3, - ) - num_patches_2, _, _, _ = calculate_h2ovl_targets( - orig_width=image_width, - orig_height=image_height, - image_size=image_processor.image_size, - target_ratios=target_ratios_2, - use_thumbnail=True, - ) - - num_patches = num_patches_1 + num_patches_2 - 1 - else: - target_ratios = self.resolve_target_ratios( - use_thumbnail=False, # Applied in calculate_targets - ) - num_patches, _, _, _ = calculate_h2ovl_targets( - orig_width=image_width, - orig_height=image_height, - image_size=image_processor.image_size, - target_ratios=target_ratios, - use_thumbnail=use_thumbnail, - ) - - return num_patches * self.image_seq_length diff --git a/vllm/transformers_utils/processors/isaac.py b/vllm/transformers_utils/processors/isaac.py deleted file mode 100644 index 7d10e9be86..0000000000 --- a/vllm/transformers_utils/processors/isaac.py +++ /dev/null @@ -1,509 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -import math -from typing import Any, TypedDict - -import numpy as np -import torch -import torch.nn.functional as F -from PIL import Image -from transformers import ( - BatchFeature, - ImageProcessingMixin, - ProcessorMixin, - TensorType, -) -from transformers.processing_utils import ProcessingKwargs -from typing_extensions import Unpack - -from vllm.tokenizers.hf import HfTokenizer - -MAX_PIXELS = 60_000_000 # 60-megapixel ceiling ≈ 8200 × 7300 px - -# Vision preprocessing constants -VISION_MEAN = (0.5, 0.5, 0.5) -VISION_STD = (0.5, 0.5, 0.5) -VISION_SCALE = 1 / 255 - - -def _make_writeable(arr: np.ndarray) -> np.ndarray: - """Return *arr* itself if it is already writeable, otherwise try to flip the - write flag in-place and finally fall back to `arr.copy()`. - This guarantees the buffer handed to `torch.from_numpy()` is always - writeable, silencing the PyTorch warning about undefined behaviour. - """ - if arr.flags.writeable: - return arr - - # First, try the cheap path — in-place flag toggle (works for mmap'd arrays - # and some shared memory buffers): - try: - arr.setflags(write=True) - return arr # success: no data copy - except ValueError: - # Buffer is inherently read-only (e.g. backed by PyAV / PIL): make copy - return arr.copy() - - -def extract_image_pil(image: Image.Image) -> torch.Tensor: - if image.width * image.height > MAX_PIXELS: - raise ValueError( - f"Image (w={image.width}, h={image.height}) > MAX=`{MAX_PIXELS}`" - ) - img = image if image.mode == "RGB" else image.convert("RGB") - arr = np.asarray(img) - arr = _make_writeable(arr) - return torch.from_numpy(arr) - - -def get_image_size_for_max_num_patches( - image_height: int, - image_width: int, - patch_size: int, - max_num_patches: int, - min_num_patches: int | None = None, - eps: float = 1e-5, - pixel_shuffle_scale: int = 1, -) -> tuple[int, int]: - r"""Compute a target resolution whose patch grid satisfies patching parametrization. - - Args: - image_height (`int`): - Height in pixels of the source image prior to any resizing. - image_width (`int`): - Width in pixels of the source image prior to any resizing. - patch_size (`int`): - Size of the square patch used by the vision encoder. - max_num_patches (`int`): - Upper bound on `(height / patch_size) * (width / patch_size)` after - resizing. - min_num_patches (`int`, *optional*): - Lower bound on the number of patches. When provided the image will - be scaled up if necessary. - eps (`float`, *optional*, defaults to 1e-5): - Convergence tolerance for the internal binary search to determine - the target dimensions. - pixel_shuffle_scale (`int`, *optional*, defaults to 1): - Additional stride multiplier applied when pixel shuffle later - reduces spatial resolution. - - Returns: - `tuple[int, int]`: Height and width (in pixels) that are multiples of - `patch_size * pixel_shuffle_scale` and respect both the maximum and - optional minimum patch-count constraints. - - """ - - def get_scaled_image_size(scale, original_size, patch_size, pixel_shuffle_scale): - scaled_size = scale * original_size - divisor = patch_size * pixel_shuffle_scale - scaled_size = math.ceil(scaled_size / divisor) * divisor - scaled_size = max(divisor, scaled_size) - return int(scaled_size) - - # Ensure divisibility - divisor = patch_size * pixel_shuffle_scale - adjusted_height = math.ceil(image_height / divisor) * divisor - adjusted_height = max(divisor, adjusted_height) - adjusted_width = math.ceil(image_width / divisor) * divisor - adjusted_width = max(divisor, adjusted_width) - - num_patches = (adjusted_height / patch_size) * (adjusted_width / patch_size) - - if min_num_patches is not None and num_patches < min_num_patches: - # Scale up - scale_min, scale_max = 1.0, 100.0 - while (scale_max - scale_min) >= eps: - scale = (scale_min + scale_max) / 2 - target_height = get_scaled_image_size( - scale, image_height, patch_size, pixel_shuffle_scale - ) - target_width = get_scaled_image_size( - scale, image_width, patch_size, pixel_shuffle_scale - ) - num_patches = (target_height / patch_size) * (target_width / patch_size) - if num_patches >= min_num_patches: - scale_max = scale - else: - scale_min = scale - scale = scale_max - target_height = get_scaled_image_size( - scale, image_height, patch_size, pixel_shuffle_scale - ) - target_width = get_scaled_image_size( - scale, image_width, patch_size, pixel_shuffle_scale - ) - return target_height, target_width - elif num_patches <= max_num_patches: - return adjusted_height, adjusted_width - else: - # Scale down - scale_min, scale_max = eps / 10, 1.0 - while (scale_max - scale_min) >= eps: - scale = (scale_min + scale_max) / 2 - target_height = get_scaled_image_size( - scale, image_height, patch_size, pixel_shuffle_scale - ) - target_width = get_scaled_image_size( - scale, image_width, patch_size, pixel_shuffle_scale - ) - num_patches = (target_height / patch_size) * (target_width / patch_size) - if num_patches <= max_num_patches: - scale_min = scale - else: - scale_max = scale - scale = scale_min - target_height = get_scaled_image_size( - scale, image_height, patch_size, pixel_shuffle_scale - ) - target_width = get_scaled_image_size( - scale, image_width, patch_size, pixel_shuffle_scale - ) - return target_height, target_width - - -_MEAN_TENSOR = torch.tensor(VISION_MEAN, dtype=torch.float32).view(1, 1, 1, -1) -_STD_TENSOR = torch.tensor(VISION_STD, dtype=torch.float32).view(1, 1, 1, -1) - - -def prepare_image_tensor( - image: torch.Tensor, - scale: float = VISION_SCALE, -) -> torch.Tensor: - r"""Standardize RGB images prior to patch extraction via rescaling and whitening. - - Args: - image (`torch.Tensor`): - Tensor with shape `(..., height, width, 3)` containing RGB values. - The tensor is converted to floating point if needed. - scale (`float`, *optional*, defaults to `VISION_SCALE`): - Scalar multiplier applied before normalization. - - Returns: - `torch.Tensor`: Normalized tensor with the same shape as the input and - dtype `torch.float32`. - - """ - if not torch.is_floating_point(image): - image = image.float() - rescaled = image * scale - - # Use precomputed tensors and move to the correct device if needed - mean_tensor = _MEAN_TENSOR.to(image.device) - std_tensor = _STD_TENSOR.to(image.device) - - normalized = (rescaled - mean_tensor) / std_tensor - return normalized - - -def patchify_vision(image: torch.Tensor, patch_size: int) -> torch.Tensor: - r"""Convert normalized images into flattened ViT-style patches. - - Args: - image (`torch.Tensor`): - Tensor of shape `(num_images, height, width, channels)`. - patch_size (`int`): - Edge length of the square patches - - Returns: - `torch.Tensor`: - Patch tensor where each position stores the flattened pixels - belonging to that patch. - - Raises: - ValueError: If `height` or `width` is not divisible by `patch_size`. - - """ - num_images, height, width, channels = image.shape - if height % patch_size or width % patch_size: - raise ValueError( - "Dimensions of images " - f"{image.shape} are not divisible by patch_size={patch_size}." - ) - patches = image.reshape( - num_images, - height // patch_size, - patch_size, - width // patch_size, - patch_size, - channels, - ) - patches = patches.permute(0, 1, 3, 2, 4, 5) - patches = patches.reshape( - num_images, - height // patch_size, - width // patch_size, - channels * patch_size * patch_size, - ) - return patches - - -def process_vision_for_patches( - images: torch.Tensor, - patch_size: int, - max_num_patches: int, - min_num_patches: int | None = None, - pixel_shuffle_scale: int = 1, -) -> tuple[torch.Tensor, list[int]]: - r"""Resize, normalize, and patchify RGB images for the vision encoder. - - Args: - images (`torch.Tensor`): - Either `(height, width, channels)` for a single image or - `(num_images, height, width, channels)` for a batch. Channels are - expected to be RGB. - patch_size (`int`): - Edge length of square patches; implicitly controls resize grid granularity. - max_num_patches (`int`): - Maximum number of patches allowed after resizing. - min_num_patches (`int`, *optional*): - Minimum number of patches. If provided, the routine upsamples images - as needed to satisfy the lower bound. - pixel_shuffle_scale (`int`, *optional*, defaults to 1): - Pixel shuffle scale factor; influences the target grid that the - function produces. - - Returns: - `tuple[torch.Tensor, list[int]]`: A pair `(patches, dims_virtual)` - where `patches` has shape `(num_images, target_h / patch_size, target_w - / patch_size, channels * patch_size**2)` and `dims_virtual` encodes - effective `(images, height, width)` dimensions after optional pixel - shuffling. - - """ - # Add batch dim if single image - if images.dim() == 3: - images = images.unsqueeze(0) - - # Permute to channel first for resize - images = images.permute(0, 3, 1, 2) - - # Get target dimensions - _, _, orig_height, orig_width = images.shape - target_height, target_width = get_image_size_for_max_num_patches( - orig_height, - orig_width, - patch_size, - max_num_patches, - min_num_patches=min_num_patches, - pixel_shuffle_scale=pixel_shuffle_scale, - ) - - # Resize - images = F.interpolate( - images, - size=(target_height, target_width), - mode="bilinear", - align_corners=False, - ) - - # Back to channel last - images = images.permute(0, 2, 3, 1) - - # Normalize - images = prepare_image_tensor(images) - - # Patchify - patches = patchify_vision(images, patch_size=patch_size) - - # Calculate dimensions for the patches - n_images, h_patches, w_patches, _ = patches.shape - dims_virtual = ( - [1, h_patches, w_patches] - if pixel_shuffle_scale == 1 - else [1, h_patches // pixel_shuffle_scale, w_patches // pixel_shuffle_scale] - ) - - return patches, dims_virtual - - -class IsaacImagesKwargs(TypedDict, total=False): - patch_size: int - max_num_patches: int - min_num_patches: int - pixel_shuffle_scale: int - - -class IsaacProcessorKwargs(ProcessingKwargs, total=False): # type: ignore[call-arg] - images_kwargs: IsaacImagesKwargs - _defaults = { - "text_kwargs": {"padding": False}, - "images_kwargs": {}, - } - - -class IsaacImageProcessor(ImageProcessingMixin): - model_input_names = ["pixel_values", "image_grid_thw"] - - def __init__( - self, - patch_size: int = 16, - vision_max_num_patches: int = 6144, - vision_min_num_patches: int = 256, - pixel_shuffle_scale: int = 2, - ) -> None: - self.patch_size = patch_size - self.vision_max_num_patches = vision_max_num_patches - self.vision_min_num_patches = vision_min_num_patches - self.pixel_shuffle_scale = pixel_shuffle_scale - - def __call__( - self, - images: Image.Image | list[Image.Image], - return_tensors: str | TensorType | None = None, - **kwargs: Unpack[IsaacImagesKwargs], - ) -> BatchFeature: - """Preprocess images into format compatible with vLLM input processing.""" - if not isinstance(images, list): - images = [images] - - all_pixel_values: list[torch.Tensor] = [] - all_image_grids: list[torch.Tensor] = [] - - for image in images: - image_tensor = extract_image_pil(image) - - patches, dims_virtual = process_vision_for_patches( - image_tensor, - patch_size=kwargs.get("patch_size", self.patch_size), - max_num_patches=kwargs.get( - "max_num_patches", self.vision_max_num_patches - ), - min_num_patches=kwargs.get( - "min_num_patches", self.vision_min_num_patches - ), - pixel_shuffle_scale=kwargs.get( - "pixel_shuffle_scale", self.pixel_shuffle_scale - ), - ) - - # Isaac packs a dummy temporal dim for images - patches = patches.unsqueeze(1) # [N, T=1, Hp, Wp, D] - - hp, wp, dim = patches.shape[-3], patches.shape[-2], patches.shape[-1] - current_num_patches = hp * wp - pixel_values = patches.reshape(current_num_patches, dim) # [N_tokens, D] - - # Use real patch dimensions for image_grid_thw, not virtual dimensions - # This ensures the vision model receives correct grid info for pixel shuffle - dims_real = [1, hp, wp] # Real patch dimensions - image_grid_thw = torch.tensor(dims_real).unsqueeze(0) - - all_pixel_values.append(pixel_values) - all_image_grids.append(image_grid_thw) - - if all_pixel_values: - final_pixel_values = torch.cat(all_pixel_values, dim=0) - final_image_grids = torch.cat(all_image_grids, dim=0) - else: - final_pixel_values = torch.empty(0, 0) - final_image_grids = torch.empty(0, 3) - - return BatchFeature( - data={ - "pixel_values": final_pixel_values, - "image_grid_thw": final_image_grids, - }, - tensor_type=return_tensors, - ) - - -class IsaacProcessor(ProcessorMixin): - attributes = ["image_processor", "tokenizer"] - - def __init__( - self, - image_processor: IsaacImageProcessor, - tokenizer: HfTokenizer, - image_token: str = "", - ): - self.image_processor = image_processor - self.tokenizer = tokenizer - - self.image_token = image_token - - def __call__( - self, - text: str | list[str] | None = None, - images: Image.Image | list[Image.Image] | None = None, - return_tensors: str | TensorType | None = None, - **kwargs: Unpack[IsaacProcessorKwargs], # type: ignore[misc] - ) -> BatchFeature: - output_kwargs = self._merge_kwargs( - IsaacProcessorKwargs, - tokenizer_init_kwargs=self.tokenizer.init_kwargs, - **kwargs, - ) - - if images is not None: - image_inputs = self.image_processor( - images, **output_kwargs["images_kwargs"] - ) - image_grid_thw = image_inputs["image_grid_thw"] - else: - image_inputs = {} - image_grid_thw = [] - - if text is not None: - if not isinstance(text, list): - text = [text] - - if image_inputs: - text = text.copy() # below lines change text in-place - merge_length = self.image_processor.pixel_shuffle_scale**2 - index = 0 - for i in range(len(text)): - while self.image_token in text[i]: - num_image_tokens = image_grid_thw[index].prod() // merge_length - text[i] = text[i].replace( - self.image_token, "<|placeholder|>" * num_image_tokens, 1 - ) - index += 1 - text[i] = text[i].replace("<|placeholder|>", "<|image_pad|>") - - text_inputs = self.tokenizer(text, **output_kwargs["text_kwargs"]) - else: - text_inputs = {} - - return BatchFeature( - data={**text_inputs, **image_inputs}, - tensor_type=return_tensors, - ) - - def apply_chat_template( - self, - messages: list[dict[str, Any]], - tokenize: bool = False, - add_generation_prompt: bool = False, - **kwargs, - ) -> Any: - # Convert mixed content messages to simple text format - processed_messages = [] - - for message in messages: - if "content" in message and isinstance(message["content"], list): - # Handle mixed content (text + image) - text_parts = [] - for content_item in message["content"]: - if content_item.get("type") == "text": - text_parts.append(content_item.get("text", "")) - elif content_item.get("type") == "image": - # Replace image with vision token - text_parts.append(self.image_token) - - processed_message = { - "role": message.get("role", "user"), - "content": "".join(text_parts), - } - processed_messages.append(processed_message) - else: - # Regular text message - processed_messages.append(message) - - kwargs["return_dict"] = False - return self.tokenizer.apply_chat_template( - processed_messages, - tokenize=tokenize, - add_generation_prompt=add_generation_prompt, - **kwargs, - ) diff --git a/vllm/transformers_utils/processors/kimi_audio.py b/vllm/transformers_utils/processors/kimi_audio.py deleted file mode 100644 index 68215c2183..0000000000 --- a/vllm/transformers_utils/processors/kimi_audio.py +++ /dev/null @@ -1,105 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -# Copyright 2026 The Moonshot AI team and the HuggingFace Inc. team. -# All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Processor for Kimi-Audio ASR model.""" - -import numpy as np -from transformers import BatchFeature, ProcessorMixin -from transformers.audio_utils import AudioInput -from transformers.tokenization_utils_base import PreTokenizedInput, TextInput - - -class KimiAudioProcessor(ProcessorMixin): - # Required for ProcessorMixin - attributes = ["feature_extractor", "tokenizer"] - feature_extractor_class = "AutoFeatureExtractor" - tokenizer_class = "AutoTokenizer" - - # Special token IDs - KIMIA_MEDIA_BEGIN: int = 151661 - KIMIA_MEDIA_END: int = 151663 - KIMIA_TEXT_BLANK: int = 151666 - - # Audio processing constants - AUDIO_SEQ_LEN: int = 376 - - def __init__(self, feature_extractor=None, tokenizer=None, **kwargs): - self.feature_extractor = feature_extractor - self.tokenizer = tokenizer - - def __call__( - self, - text: TextInput - | PreTokenizedInput - | list[TextInput] - | list[PreTokenizedInput] - | None = None, - audio: AudioInput | None = None, - return_tensors: str = "pt", - **kwargs, - ) -> BatchFeature: - if text is not None: - if not isinstance(text, list): - text = [text] - - text_inputs = self.tokenizer( - text, return_tensors=return_tensors, padding=True - ) - else: - text_inputs = {} - - if audio is not None: - # Ensure audio is a list - if isinstance(audio, np.ndarray): - audio = [audio] - - # Pad audio to hop length (required by WhisperFeatureExtractor) - hop_length = self.feature_extractor.hop_length - padded_audio = [] - for aud in audio: - length = aud.shape[-1] - if length % hop_length != 0: - pad_length = hop_length - (length % hop_length) - aud = np.pad( - aud, (0, pad_length), mode="constant", constant_values=0 - ) - padded_audio.append(aud) - - # Use feature_extractor directly like Qwen3ASR does - audio_inputs = self.feature_extractor( - padded_audio, - sampling_rate=16000, - padding=True, - return_attention_mask=True, - return_tensors=return_tensors, - ) - # Rename to match Kimi-Audio expectations - if "input_features" in audio_inputs: - audio_inputs["whisper_input_features"] = audio_inputs.pop( - "input_features" - ) - if "attention_mask" in audio_inputs: - audio_inputs["feature_attention_mask"] = audio_inputs.pop( - "attention_mask" - ) - else: - audio_inputs = {} - - return BatchFeature( - data={**text_inputs, **audio_inputs}, - tensor_type=return_tensors, - ) diff --git a/vllm/transformers_utils/processors/kimi_k25.py b/vllm/transformers_utils/processors/kimi_k25.py deleted file mode 100644 index 800dd3a7ed..0000000000 --- a/vllm/transformers_utils/processors/kimi_k25.py +++ /dev/null @@ -1,92 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from transformers import BaseImageProcessor, BatchFeature, TensorType -from transformers.processing_utils import ProcessorMixin - -from vllm.multimodal.inputs import VisionChunk -from vllm.tokenizers.hf import HfTokenizer - - -class KimiK25Processor(ProcessorMixin): - attributes = ["image_processor", "tokenizer"] - - def __init__( - self, - image_processor: BaseImageProcessor, - tokenizer: HfTokenizer, - media_token_id: int, - ) -> None: - self.image_processor = image_processor - self.tokenizer = tokenizer - - self.media_token_id = media_token_id - - def __call__( - self, - text: str | list[str] | None = None, - vision_chunks: list[VisionChunk] | None = None, - return_tensors: str | TensorType | None = None, - do_rescale: bool = True, - do_normalize: bool = True, - **kwargs, - ) -> BatchFeature: - """Args: - text: The text to be field to the model. - vision_chunks: List of `VisionChunk` items to be processed. - For image: `VisionChunkImage` with - `type='image', image=PIL.Image` - For video_chunk: `VisionChunkVideo` with - `type='video_chunk', video_chunk=list[PIL.Image]` - Returns: - [`BatchFeature`]: A [`BatchFeature`] with the following fields: - - - **input_ids** -- list of token ids to be fed to a model. - - **pixel_values** -- Pixel values to be fed to a model. - Returned when `vision_chunks` is not `None`. - - **grid_thws** -- list of image 3D grid in LLM. - Returned when `vision_chunks` is not `None`. - - """ - if vision_chunks is not None: - mm_inputs = self.image_processor.preprocess( - vision_chunks, - return_tensors=return_tensors, - do_rescale=do_rescale, - do_normalize=do_normalize, - ) - else: - mm_inputs = {} - - if text is not None: - if not isinstance(text, list): - text = [text] - - text_inputs = self.tokenizer(text) - - # Note: Modify in-place - input_ids: list[list[int]] = text_inputs["input_ids"] # type: ignore - - if vision_chunks is not None: - num_tokens_per_chunk = [ - self.image_processor.media_tokens_calculator(chunk) - for chunk in vision_chunks - ] - - for i in range(len(input_ids)): - new_input_ids = [] - for token in input_ids[i]: - if token == self.media_token_id: - new_input_ids.extend( - [self.media_token_id] * num_tokens_per_chunk.pop(0) - ) - else: - new_input_ids.append(token) - - input_ids[i] = new_input_ids - else: - text_inputs = {} - - return BatchFeature( - data={**text_inputs, **mm_inputs}, - tensor_type=return_tensors, - ) diff --git a/vllm/transformers_utils/processors/kimi_k3.py b/vllm/transformers_utils/processors/kimi_k3.py deleted file mode 100644 index e726a9c38a..0000000000 --- a/vllm/transformers_utils/processors/kimi_k3.py +++ /dev/null @@ -1,65 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from transformers import BaseImageProcessor, BatchFeature, TensorType -from transformers.processing_utils import ProcessorMixin - -from vllm.tokenizers.hf import HfTokenizer - - -class KimiK3Processor(ProcessorMixin): - """HF-style processor wrapper for the image-only Kimi-K3 model. - - K3 exposes the standard ``image`` modality, so vLLM calls this processor - with ``images=[PIL, ...]``. The underlying checkpoint image processor - (``KimiK3VisionProcessor``) works on ``{"type": "image", "image": PIL}`` - media dicts, so this wrapper adapts bare PIL images into that shape before - delegating to ``preprocess``. - - Text is only tokenized here; the single ``<|kimi_image_placeholder|>`` - token per image is expanded into the resolution-aware media block by the - model's ``_get_prompt_updates`` on the vLLM side. - """ - - attributes = ["image_processor", "tokenizer"] - - def __init__( - self, - image_processor: BaseImageProcessor, - tokenizer: HfTokenizer, - ) -> None: - self.image_processor = image_processor - self.tokenizer = tokenizer - - def __call__( - self, - text: str | list[str] | None = None, - images: object | list[object] | None = None, - return_tensors: str | TensorType | None = None, - do_rescale: bool = True, - do_normalize: bool = True, - **kwargs, - ) -> BatchFeature: - if images is not None: - if not isinstance(images, list): - images = [images] - medias = [{"type": "image", "image": image} for image in images] - mm_inputs = self.image_processor.preprocess( - medias, - return_tensors=return_tensors, - do_rescale=do_rescale, - do_normalize=do_normalize, - ) - else: - mm_inputs = {} - - if text is not None: - if not isinstance(text, list): - text = [text] - text_inputs = self.tokenizer(text) - else: - text_inputs = {} - - return BatchFeature( - data={**text_inputs, **mm_inputs}, - tensor_type=return_tensors, - ) diff --git a/vllm/transformers_utils/processors/mimo_v2_omni.py b/vllm/transformers_utils/processors/mimo_v2_omni.py deleted file mode 100644 index 0555c941b2..0000000000 --- a/vllm/transformers_utils/processors/mimo_v2_omni.py +++ /dev/null @@ -1,1234 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -# mypy: ignore-errors -"""MiMo-Omni multimodal processor for vLLM. - -Ported from SGLang's MiMoV2OmniProcessor / MiMoVLProcessor implementations. -""" - -import contextlib -import logging -import math -from collections import OrderedDict -from concurrent.futures import ThreadPoolExecutor, as_completed -from dataclasses import dataclass, field -from typing import Any, Literal - -import numpy as np -import regex as re -import torch -import torch.nn.functional as F -from PIL import Image -from transformers import BatchFeature, TensorType -from transformers.processing_utils import ProcessorMixin - -try: - import torchaudio - from torchaudio.transforms import MelSpectrogram as _MelSpectrogram - - _HAS_TORCHAUDIO = True -except ImportError: - torchaudio = None # type: ignore[assignment] - _MelSpectrogram = None # type: ignore[assignment,misc] - _HAS_TORCHAUDIO = False - -logger = logging.getLogger(__name__) - -# --------------------------------------------------------------------------- -# Constants -# --------------------------------------------------------------------------- - -_PIXEL_MEAN = [123.675, 116.28, 103.53] -_PIXEL_STD = [58.395, 57.12, 57.375] -_mean_std_cache: dict[str, tuple[torch.Tensor, torch.Tensor]] = {} - - -# --------------------------------------------------------------------------- -# Data classes -# --------------------------------------------------------------------------- - - -@dataclass -class ImageInput: - # PIL.Image | torch.Tensor (C,H,W) - image: Any - max_pixels: int | None = None - min_pixels: int | None = None - - -@dataclass -class VideoInput: - # tuple[frames_TCHW: torch.Tensor, timestamps_T: torch.Tensor] - video: Any - min_pixels: int | None = None - max_pixels: int | None = None - total_max_pixels: int | None = None - fps: float | None = None - num_frames: int | None = None - max_frames: int | None = None - min_frames: int | None = None - do_include_last_frame: bool | None = False - start_time: float | None = None - end_time: float | None = None - segment_type: Literal["individual", "partial"] = "individual" - - -@dataclass -class AudioInput: - # tuple[waveform_1D, sr] - # | np.ndarray | torch.Tensor (T,n_vq) - audio: Any - - -@dataclass -class VideoAudioInput: - video: Any # same as VideoInput.video - audio: Any # same as AudioInput.audio - min_pixels: int | None = None - max_pixels: int | None = None - total_max_pixels: int | None = None - fps: float | None = None - num_frames: int | None = None - max_frames: int | None = None - min_frames: int | None = None - do_include_last_frame: bool | None = False - start_time: float | None = None - end_time: float | None = None - segment_type: Literal["individual", "partial"] = "individual" - - -@dataclass -class Content: - type: Literal["text", "image", "video", "audio", "video_audio"] - content: Any - is_target: bool | None = None - - -@dataclass -class MiMoVLInputSample: - input_ids: torch.Tensor - labels: torch.Tensor | None - pixel_values: list[torch.Tensor] - pixel_values_videos: list[torch.Tensor] - image_thw_grids: list[torch.Tensor] - video_thw_grids: list[torch.Tensor] - audio_inputs: list[torch.Tensor] - second_per_grid_ts: list[float] = field(default_factory=list) - video_start_times: list[float] = field(default_factory=list) - audio_token_lens: list[int] = field(default_factory=list) - va_audio_inputs: list[torch.Tensor] = field(default_factory=list) - video_audio_n_segs: list[int] = field(default_factory=list) - video_audio_seg_lens: list[int] = field(default_factory=list) - position_ids: torch.Tensor | None = None - rope_deltas: torch.Tensor | None = None - extra: dict = field(default_factory=dict) - - -# --------------------------------------------------------------------------- -# Vision utilities -# --------------------------------------------------------------------------- - - -def _format_timestamp(ts: float) -> str: - return f"{int(ts // 60):02d}:{int(ts % 60):02d}" - - -def _smart_resize( - h: int, w: int, factor: int, min_px: int, max_px: int -) -> tuple[int, int]: - if min(h, w) < factor: - if h < w: - h, w = factor, int(w * factor / h) - else: - w, h = factor, int(h * factor / w) - elif max(h, w) / min(h, w) > 200: - raise ValueError(f"Aspect ratio > 200 not allowed: {h}x{w}") - h_bar = round(h / factor) * factor - w_bar = round(w / factor) * factor - if h_bar * w_bar > max_px: - beta = math.sqrt((h * w) / max_px) - h_bar = math.floor(h / beta / factor) * factor - w_bar = math.floor(w / beta / factor) * factor - elif h_bar * w_bar < min_px: - beta = math.sqrt(min_px / (h * w)) - h_bar = math.ceil(h * beta / factor) * factor - w_bar = math.ceil(w * beta / factor) * factor - return int(h_bar), int(w_bar) - - -def _standardize(images: torch.Tensor) -> torch.Tensor: - key = str(images.device) - if key not in _mean_std_cache: - mean = torch.tensor(_PIXEL_MEAN, device=images.device).view(1, -1, 1, 1) - std = torch.tensor(_PIXEL_STD, device=images.device).view(1, -1, 1, 1) - _mean_std_cache[key] = (mean, std) - mean, std = _mean_std_cache[key] - return (images - mean) / std - - -def _transform_batch( - frames: torch.Tensor, - factor: int, - min_px: int, - max_px: int, - device: torch.device | None = None, -) -> tuple[torch.Tensor, int, int]: - if device is not None: - frames = frames.to(device) - _, _, h, w = frames.shape - h_bar, w_bar = _smart_resize(h, w, factor, min_px, max_px) - resized = F.interpolate( - frames.float(), (h_bar, w_bar), mode="bilinear", align_corners=False - ) - return _standardize(resized), w_bar, h_bar - - -def _transform_single( - img: Any, - factor: int, - min_px: int, - max_px: int, - device: torch.device | None = None, -) -> tuple[torch.Tensor, int, int]: - if isinstance(img, torch.Tensor): - t = img.float() - _, h, w = t.shape - elif isinstance(img, Image.Image): - img = img.convert("RGB") - w, h = img.size - t = torch.from_numpy(np.array(img)).permute(2, 0, 1).float() - else: - raise TypeError(f"Expected Tensor or PIL.Image, got {type(img)}") - if device is not None: - t = t.to(device) - h_bar, w_bar = _smart_resize(h, w, factor, min_px, max_px) - out = F.interpolate( - t.unsqueeze(0), (h_bar, w_bar), mode="bilinear", align_corners=False - ) - return _standardize(out).squeeze(0), w_bar, h_bar - - -# --------------------------------------------------------------------------- -# Core processor -# --------------------------------------------------------------------------- - - -class MiMoVLProcessor: - """Core MiMo-VL multimodal processor. - - Handles image/video/audio preprocessing and token sequence construction. - Ported from SGLang's MiMoVLProcessor. - - Media strings (URL / path / data:) are resolved upstream in vLLM's media - pipeline; this processor accepts only already-decoded inputs (``PIL.Image`` - for images, ``(waveform, sr)`` tuples for audio). - """ - - def __init__( - self, - tokenizer: Any, - patch_size: int = 14, - merge_size: int = 2, - temporal_patch_size: int = 2, - temporal_compression_ratio: int = 1, - use_video_timestamps: bool = True, - video_audio_interleave_length: int = 0, - audio_kernel_size: int = 3, - audio_stride_size: int = 2, - audio_avg_pooler: int = 2, - audio_sampling_rate: int = 24000, - audio_nfft: int = 960, - audio_hop_length: int = 240, - audio_window_size: int = 960, - audio_fmin: float = 0.0, - audio_fmax: float | None = None, - audio_n_mels: int = 128, - audio_segment_size: int = 6000, - audio_channels: int = 8, - audio_group_size: int = 4, - audio_input_id_per_second: float = 25.0, - audio_zeroemb_idx: int = 4096, - image_min_pixels: int | None = None, - image_max_pixels: int | None = None, - video_min_pixels: int | None = None, - video_max_pixels: int | None = None, - video_total_max_pixels: int | None = None, - fps: float | None = None, - num_frames: int | None = None, - max_frames: int | None = None, - min_frames: int | None = None, - image_token_id: int | None = None, - video_token_id: int | None = None, - audio_token_id: int | None = None, - vision_start_token_id: int | None = None, - vision_end_token_id: int | None = None, - audio_start_token_id: int | None = None, - audio_end_token_id: int | None = None, - video_start_token_id: int | None = None, - video_end_token_id: int | None = None, - pad_token_id: int | None = None, - rope_type: str = "rope", - video_process_num_threads: int = 16, - device: Any | None = None, - **kwargs: Any, - ) -> None: - self.tokenizer = tokenizer - self.video_process_num_threads = video_process_num_threads - self.device = torch.device(device) if isinstance(device, str) else device - - self.rope_type = "rope" if rope_type == "1d" else rope_type - assert self.rope_type in ("rope", "mrope"), ( - f"Unknown rope_type: {self.rope_type}" - ) - - # video timestamps require 1-D rope - assert use_video_timestamps, "use_video_timestamps must be True" - assert self.rope_type == "rope", ( - "use_video_timestamps requires rope_type='rope'" - ) - self.use_video_timestamps = use_video_timestamps - self.video_audio_interleave_length = video_audio_interleave_length - - self.image_token_id = image_token_id - self.video_token_id = video_token_id - self.audio_token_id = audio_token_id - self.vision_start_token_id = vision_start_token_id - self.vision_end_token_id = vision_end_token_id - self.audio_start_token_id = audio_start_token_id - self.audio_end_token_id = audio_end_token_id - self.video_start_token_id = video_start_token_id - self.video_end_token_id = video_end_token_id - self.pad_token_id = pad_token_id - - self.patch_size = patch_size - self.merge_size = merge_size - self.temporal_patch_size = temporal_patch_size - self.temporal_compression_ratio = temporal_compression_ratio - - self.audio_sampling_rate = audio_sampling_rate - self.audio_nfft = audio_nfft - self.audio_hop_length = audio_hop_length - self.audio_window_size = audio_window_size - self.audio_fmin = audio_fmin - self.audio_fmax = audio_fmax - self.audio_n_mels = audio_n_mels - self.audio_segment_size = audio_segment_size - self.audio_kernel_size = audio_kernel_size - self.audio_stride_size = audio_stride_size - self.audio_avg_pooler = audio_avg_pooler - self.audio_channels = audio_channels - self.audio_group_size = audio_group_size - self.audio_input_id_per_second = audio_input_id_per_second - - self._mel_spec_kwargs = dict( - sample_rate=audio_sampling_rate, - n_fft=audio_nfft, - hop_length=audio_hop_length, - win_length=audio_window_size, - f_min=audio_fmin, - f_max=audio_fmax, - n_mels=audio_n_mels, - power=1.0, - center=True, - ) - self._mel_spectrogram: Any | None = None - self._resamplers: OrderedDict = OrderedDict() - self._resamplers_max = 16 - - if isinstance(audio_zeroemb_idx, int): - self.audio_zeroemb_idxs = torch.tensor( - [audio_zeroemb_idx] * audio_channels, dtype=torch.int32 - ) - else: - self.audio_zeroemb_idxs = torch.tensor(audio_zeroemb_idx, dtype=torch.int32) - - assert image_min_pixels is not None, "image_min_pixels must be set" - assert image_max_pixels is not None, "image_max_pixels must be set" - assert video_min_pixels is not None, "video_min_pixels must be set" - assert video_max_pixels is not None, "video_max_pixels must be set" - assert video_total_max_pixels is not None, "video_total_max_pixels must be set" - assert fps is not None or num_frames is not None, ( - "fps or num_frames must be set" - ) - - self._img_kw = {"min_pixels": image_min_pixels, "max_pixels": image_max_pixels} - self._vid_kw = { - "min_pixels": video_min_pixels, - "max_pixels": video_max_pixels, - "total_max_pixels": video_total_max_pixels, - "fps": fps, - "num_frames": num_frames, - "max_frames": max_frames, - "min_frames": min_frames, - } - - @property - def mel_spectrogram(self) -> Any: - if self._mel_spectrogram is None: - if _MelSpectrogram is None: - raise RuntimeError( - "torchaudio is required for audio. " - "Install with: pip install torchaudio" - ) - self._mel_spectrogram = _MelSpectrogram(**self._mel_spec_kwargs) - return self._mel_spectrogram - - def _resolve_img_kw(self, img: ImageInput) -> dict: - return { - "min_px": ( - img.min_pixels - if img.min_pixels is not None - else self._img_kw["min_pixels"] - ), - "max_px": ( - img.max_pixels - if img.max_pixels is not None - else self._img_kw["max_pixels"] - ), - } - - def _resolve_vid_kw(self, vid: VideoInput) -> dict: - kw: dict = {} - for k in ("min_pixels", "max_pixels", "total_max_pixels"): - kw[k] = getattr(vid, k) or self._vid_kw[k] - if vid.num_frames is not None: - kw["num_frames"] = vid.num_frames - elif vid.fps is not None: - kw["fps"] = vid.fps - if vid.max_frames is not None: - kw["max_frames"] = vid.max_frames - if vid.min_frames is not None: - kw["min_frames"] = vid.min_frames - elif self._vid_kw["num_frames"] is not None: - kw["num_frames"] = self._vid_kw["num_frames"] - elif self._vid_kw["fps"] is not None: - kw["fps"] = self._vid_kw["fps"] - if self._vid_kw["max_frames"] is not None: - kw["max_frames"] = self._vid_kw["max_frames"] - if self._vid_kw["min_frames"] is not None: - kw["min_frames"] = self._vid_kw["min_frames"] - else: - raise ValueError( - "No video sampling strategy specified (fps or num_frames)." - ) - return kw - - def preprocess_audio(self, audio: Any) -> tuple[torch.Tensor, int]: - """Convert a pre-loaded ``(waveform, sr)`` tuple into (mel_spec, token_len).""" - if not isinstance(audio, tuple): - raise ValueError( - f"Unsupported audio source type: {type(audio)}. Audio must be a " - "pre-decoded (waveform, sample_rate) tuple; URL/path/bytes " - "resolution is handled upstream in vLLM's media pipeline." - ) - waveform, original_sr = audio - - if original_sr != self.audio_sampling_rate: - if original_sr not in self._resamplers: - if len(self._resamplers) >= self._resamplers_max: - self._resamplers.popitem(last=False) - self._resamplers[original_sr] = torchaudio.transforms.Resample( - orig_freq=original_sr, new_freq=self.audio_sampling_rate - ) - self._resamplers.move_to_end(original_sr) - waveform = self._resamplers[original_sr](waveform) - - if waveform.ndim == 2: - waveform = waveform.mean(dim=0) - spec = self.mel_spectrogram(waveform[None, :]) - spec = torch.log(torch.clip(spec, min=1e-7)).squeeze().transpose(0, 1) - - n = spec.shape[0] - n = n + 3 - self.audio_kernel_size - n = (n + 2 - self.audio_kernel_size) // self.audio_stride_size + 1 - n = n // self.audio_avg_pooler + int(n % self.audio_avg_pooler != 0) - token_len = math.ceil(n / self.audio_group_size) - return spec, token_len - - def process_image(self, image: ImageInput) -> torch.Tensor: - kw = self._resolve_img_kw(image) - src = image.image - tensor, _, _ = _transform_single( - src, - factor=self.patch_size * self.merge_size, - device=self.device, - **kw, - ) - return tensor - - def process_video( - self, video_input: VideoInput - ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, dict]: - kw = self._resolve_vid_kw(video_input) - video = video_input.video - if not isinstance(video, tuple): - raise ValueError( - f"video must be a (frames_TCHW, timestamps_T) tuple, " - f"got {type(video)}. " - "Decode the video before calling the processor." - ) - frames, timestamps = video - - fps = ( - 1.0 - if len(timestamps) < 2 - else float(1.0 / (float(timestamps[1]) - float(timestamps[0]))) - ) - start = ( - video_input.start_time - if video_input.start_time is not None - else float(timestamps[0]) - ) - end = ( - video_input.end_time - if video_input.end_time is not None - else float(timestamps[-1]) + 1.0 / fps - ) - - if video_input.segment_type != "individual": - mask = (timestamps >= start) & (timestamps < end) - idxs = torch.where(mask)[0] - if len(idxs) == 0: - idxs = torch.where(timestamps <= start)[0][-1:] - frames, timestamps = frames[idxs], timestamps[idxs] - - tp = self.temporal_patch_size * self.temporal_compression_ratio - n = frames.shape[0] - total_px = kw["total_max_pixels"] - max_px = max( - kw["min_pixels"], min(total_px * tp // max(n, 1), kw["max_pixels"]) - ) - - if n % tp != 0: - pad = tp - n % tp - frames = torch.cat( - [frames, frames[-1:].repeat(pad, *([1] * (frames.ndim - 1)))], - dim=0, - ) - timestamps = torch.cat([timestamps, timestamps[-1:].repeat(pad)], dim=0) - - transformed, _, _ = _transform_batch( - frames, - factor=self.patch_size * self.merge_size, - min_px=kw["min_pixels"], - max_px=max_px, - device=self.device, - ) - patches, thw = self._flatten_visual(transformed, "video") - meta = { - "fps_sampled": fps, - "segment_start_time": start, - "segment_end_time": end, - } - return patches, thw, timestamps, meta - - def process_audio(self, audio: AudioInput) -> Any: - src = audio.audio - if isinstance(src, np.ndarray): - src = (torch.from_numpy(src).float(), self.audio_sampling_rate) - if isinstance(src, tuple): - return self.preprocess_audio(src) - # Pre-tokenized tensor (T, n_vq) - assert isinstance(src, torch.Tensor) and src.ndim == 2 - T = src.shape[0] - src = src[:, : self.audio_channels].to(torch.long) - pad_T = ( - (T + self.audio_group_size - 1) - // self.audio_group_size - * self.audio_group_size - ) - padding = ( - torch.zeros(pad_T - T, self.audio_channels, dtype=torch.long) + src[-1] - ) - src = torch.cat([src, padding], dim=0) - return src.reshape( - pad_T // self.audio_group_size, self.audio_group_size, self.audio_channels - ) - - def _flatten_visual( - self, visual: torch.Tensor, kind: str - ) -> tuple[torch.Tensor, torch.Tensor]: - if kind == "image": - h, w = visual.shape[-2:] - patches = visual.unsqueeze(0).repeat(self.temporal_patch_size, 1, 1, 1) - else: # video / video_audio - temporal_stride = self.temporal_compression_ratio * self.temporal_patch_size - assert visual.shape[0] % temporal_stride == 0 - patches = visual - h, w = patches.shape[-2:] - - C = patches.shape[1] - grid_t = patches.shape[0] // self.temporal_patch_size - grid_h, grid_w = h // self.patch_size, w // self.patch_size - - patches = ( - patches.contiguous() - .view( - grid_t, - self.temporal_patch_size, - C, - grid_h // self.merge_size, - self.merge_size, - self.patch_size, - grid_w // self.merge_size, - self.merge_size, - self.patch_size, - ) - .permute(0, 3, 6, 4, 7, 2, 1, 5, 8) - .contiguous() - .view( - grid_t * grid_h * grid_w, - C * self.temporal_patch_size * self.patch_size * self.patch_size, - ) - ) - thw = torch.tensor([grid_t, grid_h, grid_w], dtype=torch.int32) - return patches, thw - - def process( - self, contents: list[Content], verbose: bool = False - ) -> MiMoVLInputSample: - input_ids: list[int] = [] - labels: list[int] = [] - img_pv: list[torch.Tensor] = [] - img_grids: list[torch.Tensor] = [] - vid_pv: list[torch.Tensor] = [] - vid_grids: list[torch.Tensor] = [] - audio_inputs: list[torch.Tensor] = [] - is_audio_tokenized: list[bool] = [] - audio_token_lens: list[int] = [] - second_per_grid_ts: list[float] = [] - video_start_times: list[float] = [] - va_audio_inputs: list[torch.Tensor] = [] - video_audio_n_segs: list[int] = [] - video_audio_seg_lens: list[int] = [] - - # Pre-decode videos in parallel - vid_info = [ - (i, c.content, c.type == "video_audio") - for i, c in enumerate(contents) - if c.type in ("video", "video_audio") - ] - vid_results: dict[int, tuple] = {} - if vid_info: - n_t = min(self.video_process_num_threads, len(vid_info)) - if n_t > 1 and len(vid_info) > 1: - with ThreadPoolExecutor(max_workers=n_t) as ex: - fut_map = { - ex.submit(self.process_video, vi): idx - for idx, vi, _ in vid_info - } - for fut in as_completed(fut_map): - vid_results[fut_map[fut]] = fut.result() - else: - for idx, vi, _ in vid_info: - vid_results[idx] = self.process_video(vi) - - for ci, content in enumerate(contents): - _ids: list[int] = [] - _lbls: list[int] | None = None - - if content.type == "text": - _ids = ( - self.tokenizer.encode(content.content) - if isinstance(content.content, str) - else list(content.content) - ) - if content.is_target: - _lbls = _ids - - elif content.type == "image": - tensor = self.process_image(content.content) - patches, thw = self._flatten_visual(tensor, "image") - t, h, w = thw.tolist() - n_tok = (t * h * w) // (self.merge_size**2) - img_pv.append(patches) - img_grids.append(thw) - _ids = ( - [self.vision_start_token_id] - + [self.image_token_id] * n_tok - + [self.vision_end_token_id] - ) - - elif content.type == "video": - patches, thw, ts, meta = vid_results[ci] - t, h, w = thw.tolist() - n_per_grid = h * w // (self.merge_size**2) - vid_pv.append(patches) - vid_grids.append(thw) - second_per_grid_ts.append( - self.temporal_patch_size / meta["fps_sampled"] - ) - video_start_times.append(float(ts[0])) - video_audio_n_segs.append(0) - - stride = self.temporal_patch_size * self.temporal_compression_ratio - ts_texts = [_format_timestamp(float(x)) for x in ts[::stride]] - ts_ids_list = [self.tokenizer.encode(s) for s in ts_texts] - - _ids = [self.video_start_token_id] - for ts_ids in ts_ids_list: - _ids += ( - ts_ids - + [self.vision_start_token_id] - + [self.video_token_id] * n_per_grid - + [self.vision_end_token_id] - ) - _ids += [self.video_end_token_id] - - elif content.type == "audio": - processed = self.process_audio(content.content) - if isinstance(processed, tuple): - is_audio_tokenized.append(False) - spec, tok_len = processed - audio_inputs.append(spec) - else: - is_audio_tokenized.append(True) - tok_len = processed.shape[0] - audio_inputs.append(processed) - audio_token_lens.append(tok_len) - _ids = ( - [self.audio_start_token_id] - + [self.audio_token_id] * tok_len - + [self.audio_end_token_id] - ) - - elif content.type == "video_audio": - patches, thw, ts, meta = vid_results[ci] - second_per_grid_ts.append( - self.temporal_patch_size / meta["fps_sampled"] - ) - video_start_times.append(float(ts[0])) - processed_audio = self.process_audio(content.content) - tok_per_sec = self.audio_input_id_per_second / self.audio_group_size - - t, h, w = thw.tolist() - vid_pv.append(patches) - vid_grids.append(thw) - - if isinstance(processed_audio, tuple): - # Mel spec (not pre-tokenized): store in va_audio_inputs separately - spec, total_atok = processed_audio - va_audio_inputs.append(spec) - _va_is_tokenized = False - else: - # Pre-tokenized: not expected in vLLM, but handle defensively - total_atok = processed_audio.shape[0] - _va_is_tokenized = True - - n_per_grid = h * w // (self.merge_size**2) - stride = self.temporal_patch_size * self.temporal_compression_ratio - grid_ts = ts[::stride] - ts_texts = [_format_timestamp(float(x)) for x in grid_ts] - ts_ids_list = [self.tokenizer.encode(s) for s in ts_texts] - - units: list[tuple] = [] - for i in range(len(grid_ts)): - a_start = int(float(grid_ts[i]) * tok_per_sec) - a_end = ( - int(float(grid_ts[i + 1]) * tok_per_sec) - if i < len(grid_ts) - 1 - else int(meta["segment_end_time"] * tok_per_sec) - ) - seg_len = min(a_end, total_atok) - a_start - assert seg_len > 0, f"Zero-length audio segment at grid index {i}" - seg = ( - processed_audio[a_start : a_start + seg_len] - if _va_is_tokenized - else None - ) - units.append( - ( - float(grid_ts[i]), - ts_texts[i], - ts_ids_list[i], - n_per_grid, - seg_len, - seg, - ) - ) - - il = self.video_audio_interleave_length - if il == -1: - groups: list[list] = [list(enumerate(units))] - elif il == 0: - groups = [[(i, u)] for i, u in enumerate(units)] - else: - groups, cur, t_ptr = [], [], 0.0 - for i, u in enumerate(units): - while u[0] >= t_ptr + il: - if cur: - groups.append(cur) - cur = [] - t_ptr += il - cur.append((i, u)) - if cur: - groups.append(cur) - - # Track n_segs (= num groups) and per-group audio token counts - video_audio_n_segs.append(len(groups)) - for group in groups: - group_seg_len = sum(u[4] for _, u in group) - video_audio_seg_lens.append(group_seg_len) - - _ids = [self.video_start_token_id] - for group in groups: - _ids += group[0][1][2] # first-unit timestamp token ids - _vid_tok: list[int] = [] - _aud_tok: list[int] = [] - for _, u in group: - _, _, _, vid_n, seg_n, seg_audio = u - _vid_tok += ( - [self.vision_start_token_id] - + [self.video_token_id] * vid_n - + [self.vision_end_token_id] - ) - _aud_tok += [self.audio_token_id] * seg_n - if seg_audio is not None: - # Pre-tokenized per-frame segments (rare in vLLM) - audio_inputs.append(seg_audio) - _ids += ( - _vid_tok - + [self.audio_start_token_id] - + _aud_tok - + [self.audio_end_token_id] - ) - _ids += [self.video_end_token_id] - - input_ids.extend(_ids) - labels.extend( - _lbls if _lbls is not None else [self.pad_token_id] * len(_ids) - ) - - ids_t = torch.tensor(input_ids) - lbl_arr = np.roll(labels, shift=-1) - lbl_arr[-1] = self.pad_token_id - lbl_t = torch.tensor(lbl_arr) - - extra: dict = {} - if is_audio_tokenized: - assert all(is_audio_tokenized) or not any(is_audio_tokenized) - extra["is_audio_tokenized"] = is_audio_tokenized[0] - - position_ids = torch.arange(ids_t.shape[0]).expand(3, -1) - rope_deltas = torch.zeros((1, 1), dtype=torch.int32) - - return MiMoVLInputSample( - input_ids=ids_t, - labels=lbl_t, - pixel_values=img_pv, - pixel_values_videos=vid_pv, - image_thw_grids=img_grids, - video_thw_grids=vid_grids, - audio_inputs=audio_inputs, - second_per_grid_ts=second_per_grid_ts, - video_start_times=video_start_times, - audio_token_lens=audio_token_lens, - va_audio_inputs=va_audio_inputs, - video_audio_n_segs=video_audio_n_segs, - video_audio_seg_lens=video_audio_seg_lens, - position_ids=position_ids, - rope_deltas=rope_deltas, - extra=extra, - ) - - -# --------------------------------------------------------------------------- -# vLLM ProcessorMixin wrapper -# --------------------------------------------------------------------------- - - -class MiMoOmniProcessor(ProcessorMixin): - """HuggingFace-compatible ProcessorMixin wrapper for MiMo-Omni. - - Accepts PIL images, pre-decoded video tuples (frames_TCHW, timestamps_T), - and audio (file path / (waveform, sr) tuple / numpy array). - """ - - attributes = ["tokenizer"] - tokenizer_class = "AutoTokenizer" - - # Single or multi-pad placeholders produced by the chat template / prior expansion - _IMG_RE = re.compile(r"<\|vision_start\|>(?:<\|image_pad\|>)+<\|vision_end\|>") - _VID_RE = re.compile(r"<\|vision_start\|>(?:<\|video_pad\|>)+<\|vision_end\|>") - _AUD_RE = re.compile( - r"<\|mimo_audio_start\|>(?:<\|audio_pad\|>)+<\|mimo_audio_end\|>" - ) - - _MM_RE = re.compile( - r"(<\|vision_start\|>(?:<\|image_pad\|>)+<\|vision_end\|>" - r"|<\|vision_start\|>(?:<\|video_pad\|>)+<\|vision_end\|>" - r"|<\|mimo_audio_start\|>(?:<\|audio_pad\|>)+<\|mimo_audio_end\|>)" - ) - - def __init__( - self, - tokenizer: Any, - *, - patch_size: int = 14, - merge_size: int = 2, - temporal_patch_size: int = 2, - temporal_compression_ratio: int = 1, - image_min_pixels: int | None = None, - image_max_pixels: int | None = None, - video_min_pixels: int | None = None, - video_max_pixels: int | None = None, - video_total_max_pixels: int | None = None, - fps: float = 2.0, - num_frames: int | None = None, - max_frames: int = 256, - min_frames: int = 8, - video_audio_interleave_length: int = 0, - audio_sampling_rate: int = 24000, - audio_nfft: int = 960, - audio_hop_length: int = 240, - audio_window_size: int = 960, - audio_fmin: float = 0.0, - audio_fmax: float | None = None, - audio_n_mels: int = 128, - audio_segment_size: int = 6000, - audio_kernel_size: int = 3, - audio_stride_size: int = 2, - audio_avg_pooler: int = 2, - audio_channels: int = 8, - audio_group_size: int = 4, - audio_input_id_per_second: float = 25.0, - audio_zeroemb_idx: int = 4096, - image_token_id: int | None = None, - video_token_id: int | None = None, - audio_token_id: int | None = None, - vision_start_token_id: int | None = None, - vision_end_token_id: int | None = None, - audio_start_token_id: int | None = None, - audio_end_token_id: int | None = None, - video_start_token_id: int | None = None, - video_end_token_id: int | None = None, - rope_type: str = "rope", - ) -> None: - self.tokenizer = tokenizer - - unit = patch_size * merge_size - self.mimo_processor = MiMoVLProcessor( - tokenizer=tokenizer, - patch_size=patch_size, - merge_size=merge_size, - temporal_patch_size=temporal_patch_size, - temporal_compression_ratio=temporal_compression_ratio, - use_video_timestamps=True, - video_audio_interleave_length=video_audio_interleave_length, - audio_sampling_rate=audio_sampling_rate, - audio_nfft=audio_nfft, - audio_hop_length=audio_hop_length, - audio_window_size=audio_window_size, - audio_fmin=audio_fmin, - audio_fmax=audio_fmax, - audio_n_mels=audio_n_mels, - audio_segment_size=audio_segment_size, - audio_kernel_size=audio_kernel_size, - audio_stride_size=audio_stride_size, - audio_avg_pooler=audio_avg_pooler, - audio_channels=audio_channels, - audio_group_size=audio_group_size, - audio_input_id_per_second=audio_input_id_per_second, - audio_zeroemb_idx=audio_zeroemb_idx, - image_min_pixels=image_min_pixels or (4 * unit * unit), - image_max_pixels=image_max_pixels or (4096 * unit * unit), - video_min_pixels=video_min_pixels or (4 * unit * unit), - video_max_pixels=video_max_pixels or (4096 * unit * unit), - video_total_max_pixels=video_total_max_pixels or (16384 * unit * unit), - fps=fps, - num_frames=num_frames, - max_frames=max_frames, - min_frames=min_frames, - image_token_id=image_token_id, - video_token_id=video_token_id, - audio_token_id=audio_token_id, - vision_start_token_id=vision_start_token_id, - vision_end_token_id=vision_end_token_id, - audio_start_token_id=audio_start_token_id, - audio_end_token_id=audio_end_token_id, - video_start_token_id=video_start_token_id, - video_end_token_id=video_end_token_id, - pad_token_id=tokenizer.pad_token_id, - rope_type=rope_type, - ) - - @classmethod - def from_hf_config( - cls, - tokenizer: Any, - hf_config: Any, - ) -> "MiMoOmniProcessor": - """Instantiate directly from an HF model config object.""" - vc = hf_config.vision_config - if isinstance(vc, dict): - patch_size = vc.get("patch_size", 14) - merge_size = vc.get("spatial_merge_size", 2) - temporal_patch_size = vc.get("temporal_patch_size", 2) - else: - patch_size = getattr(vc, "patch_size", 14) - merge_size = getattr(vc, "spatial_merge_size", 2) - temporal_patch_size = getattr(vc, "temporal_patch_size", 2) - - pc: dict = getattr(hf_config, "processor_config", {}) or {} - ac = getattr(hf_config, "audio_config", None) - audio_sr: int | None = pc.get("audio_sampling_rate") - if audio_sr is None and ac is not None: - if isinstance(ac, dict): - audio_sr = ac.get("sampling_rate") or ac.get("sample_rate") - else: - audio_sr = getattr(ac, "sampling_rate", None) or getattr( - ac, "sample_rate", None - ) - - rope_type = "rope" - rs = getattr(hf_config, "rope_scaling", None) - if rs and rs.get("type") == "default" and rs.get("mrope_section") is not None: - rope_type = "mrope" - - unit = patch_size * merge_size - return cls( - tokenizer, - patch_size=patch_size, - merge_size=merge_size, - temporal_patch_size=temporal_patch_size, - image_min_pixels=pc.get("image_min_pixels") or (4 * unit * unit), - image_max_pixels=pc.get("image_max_pixels") or (4096 * unit * unit), - video_min_pixels=pc.get("video_min_pixels") or (4 * unit * unit), - video_max_pixels=pc.get("video_max_pixels") or (4096 * unit * unit), - video_total_max_pixels=( - pc.get("video_total_max_pixels") or (16384 * unit * unit) - ), - fps=pc.get("fps") or 2.0, - num_frames=pc.get("num_frames"), - max_frames=pc.get("max_frames") or 256, - min_frames=pc.get("min_frames") or 8, - video_audio_interleave_length=pc.get("video_audio_interleave_length", 0), - audio_sampling_rate=audio_sr or 24000, - image_token_id=pc.get("image_token_id"), - video_token_id=pc.get("video_token_id"), - audio_token_id=pc.get("audio_token_id"), - vision_start_token_id=pc.get("vision_start_token_id"), - vision_end_token_id=pc.get("vision_end_token_id"), - audio_start_token_id=pc.get("audio_start_token_id"), - audio_end_token_id=pc.get("audio_end_token_id"), - video_start_token_id=pc.get("video_start_token_id"), - video_end_token_id=pc.get("video_end_token_id"), - rope_type=rope_type, - ) - - @property - def image_token(self) -> str: - """Token string used as image placeholder (for vLLM integration).""" - return "<|image_pad|>" - - @property - def video_token(self) -> str: - """Token string used as video placeholder (for vLLM integration).""" - return "<|video_pad|>" - - @property - def image_processor(self) -> Any: - """Minimal image-processor-like object for vLLM processing-info compat.""" - p = self.mimo_processor - - class _ImageProcessor: - merge_size = p.merge_size - size = { - "shortest_edge": p._img_kw["min_pixels"], - "longest_edge": p._img_kw["max_pixels"], - } - - return _ImageProcessor() - - def _modality(self, token: str) -> str: - if self._IMG_RE.fullmatch(token): - return "image" - if self._VID_RE.fullmatch(token): - return "video" - if self._AUD_RE.fullmatch(token): - return "audio" - return "unknown" - - def __call__( - self, - text: str | list[str] | None = None, - images: Any = None, - videos: Any = None, - audio: Any = None, - video_audio: Any = None, - return_tensors: str | TensorType | None = None, - **kwargs: Any, - ) -> BatchFeature: - """Process multimodal inputs into model-ready tensors. - - Args: - text: Prompt string(s) containing multimodal placeholders - ``<|vision_start|><|image_pad|><|vision_end|>``, - ``<|vision_start|><|video_pad|><|vision_end|>``, or - ``<|mimo_audio_start|><|audio_pad|><|mimo_audio_end|>``. - images: PIL.Image or list[PIL.Image]. - videos: list of ``(frames_TCHW: torch.Tensor, timestamps_T: torch.Tensor)`` - tuples (pre-decoded). - audio: list of ``str`` (path/url/base64), ``bytes``, - ``(waveform_1D, sample_rate)`` tuples, or ``np.ndarray``. - video_audio: audio track extracted from ``videos``, in the same - formats accepted by ``audio``. - return_tensors: Passed to :class:`BatchFeature`. - - Returns: - :class:`BatchFeature` with keys: - - ``input_ids`` - - ``pixel_values`` + ``image_grid_thw`` - - ``pixel_values_videos`` + ``video_grid_thw`` + ``second_per_grid_ts`` - - ``audio_features`` - - """ - if isinstance(text, list): - text = text[0] if len(text) == 1 else "\n".join(text) - - imgs: list = ( - ([images] if isinstance(images, Image.Image) else list(images)) - if images is not None - else [] - ) - vids: list = list(videos) if videos is not None else [] - auds: list = list(audio) if audio is not None else [] - va_items: list = list(video_audio) if video_audio is not None else [] - - # If audio exists but text has no audio placeholder, prepend one - _aud_placeholder = "<|mimo_audio_start|><|audio_pad|><|mimo_audio_end|>" - if auds and text is not None and not self._AUD_RE.search(text): - text = _aud_placeholder + text - - # Build Content list - contents: list[Content] = [] - - if text and (imgs or vids or auds or va_items): - parts = self._MM_RE.split(text) - img_it = iter(imgs) - vid_it = iter(vids) - aud_it = iter(auds) - va_it = iter(va_items) - for part in parts: - if self._MM_RE.fullmatch(part): - mod = self._modality(part) - if mod == "image": - with contextlib.suppress(StopIteration): - contents.append( - Content( - type="image", - content=ImageInput(image=next(img_it)), - ) - ) - elif mod == "video": - # Try regular video first, fall back to video_audio - vid_item = None - vid_type = "video" - with contextlib.suppress(StopIteration): - vid_item = next(vid_it) - if vid_item is None: - with contextlib.suppress(StopIteration): - vid_item = next(va_it) - vid_type = "video_audio" - if vid_item is not None: - if vid_type == "video": - contents.append( - Content( - type="video", - content=VideoInput(video=vid_item), - ) - ) - else: - contents.append( - Content( - type="video_audio", - content=vid_item, - ) - ) - elif mod == "audio": - with contextlib.suppress(StopIteration): - contents.append( - Content( - type="audio", - content=AudioInput(audio=next(aud_it)), - ) - ) - elif part: - contents.append(Content(type="text", content=part)) - elif text: - contents.append(Content(type="text", content=text)) - else: - for img in imgs: - contents.append(Content(type="image", content=ImageInput(image=img))) - for vid in vids: - contents.append(Content(type="video", content=VideoInput(video=vid))) - for aud in auds: - contents.append(Content(type="audio", content=AudioInput(audio=aud))) - for va_item in va_items: - contents.append(Content(type="video_audio", content=va_item)) - - if not contents: - ids = self.tokenizer(text or "", return_tensors=return_tensors)["input_ids"] - return BatchFeature(data={"input_ids": ids}, tensor_type=return_tensors) - - sample = self.mimo_processor.process(contents, verbose=False) - - # vLLM expects input_ids to have a batch dimension [1, seq_len]. - data: dict = {"input_ids": sample.input_ids.unsqueeze(0)} - - if sample.pixel_values: - data["pixel_values"] = torch.cat(sample.pixel_values, dim=0) - data["image_grid_thw"] = torch.stack(sample.image_thw_grids) - - if sample.pixel_values_videos: - data["pixel_values_videos"] = torch.cat(sample.pixel_values_videos, dim=0) - data["video_grid_thw"] = torch.stack(sample.video_thw_grids) - if sample.second_per_grid_ts: - data["second_per_grid_ts"] = torch.tensor( - sample.second_per_grid_ts, dtype=torch.float32 - ) - if sample.video_start_times: - data["video_start_times"] = torch.tensor( - sample.video_start_times, dtype=torch.float32 - ) - if sample.video_audio_n_segs: - data["video_audio_n_segs"] = torch.tensor( - sample.video_audio_n_segs, dtype=torch.long - ) - # video_audio_seg_lens: 2D padded tensor (num_videos, max_T). - # Row i has the per-group audio token lengths for video i - # (zeros for regular videos; valid values for video_audio videos). - n_segs_list = sample.video_audio_n_segs - max_segs = max(n_segs_list) if n_segs_list else 0 - if max_segs > 0: - seg_lens_2d = torch.zeros(len(n_segs_list), max_segs, dtype=torch.long) - flat_cursor = 0 - for vi, n in enumerate(n_segs_list): - if n > 0: - seg_lens_2d[vi, :n] = torch.tensor( - sample.video_audio_seg_lens[flat_cursor : flat_cursor + n], - dtype=torch.long, - ) - flat_cursor += n - data["video_audio_seg_lens"] = seg_lens_2d - - # audio_features is a list of variable-length mel-spec tensors; pop it - # before BatchFeature conversion to avoid "batched tensors of the same - # length" errors, then re-attach it after. - audio_features = None - if sample.audio_inputs: - audio_features = sample.audio_inputs - if "is_audio_tokenized" in sample.extra: - data["is_audio_tokenized"] = sample.extra["is_audio_tokenized"] - if sample.audio_token_lens: - data["audio_token_lens"] = torch.tensor( - sample.audio_token_lens, dtype=torch.long - ) - - bf = BatchFeature(data=data, tensor_type=return_tensors) - if audio_features is not None: - bf["audio_features"] = audio_features - # va_audio_features: list of mel-spec tensors (one per video_audio item) - if sample.va_audio_inputs: - bf["va_audio_features"] = sample.va_audio_inputs - return bf diff --git a/vllm/transformers_utils/processors/minicpmo.py b/vllm/transformers_utils/processors/minicpmo.py deleted file mode 100644 index 17a915b2d6..0000000000 --- a/vllm/transformers_utils/processors/minicpmo.py +++ /dev/null @@ -1,523 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -# coding=utf-8 -# Copyright 2025 The OpenBMB Team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Processor class for MiniCPMO.""" - -import math -from typing import TypeAlias - -import numpy as np -import regex -import torch -from transformers.image_processing_utils import BatchFeature -from transformers.image_utils import ImageInput -from transformers.processing_utils import ProcessorMixin -from transformers.tokenization_utils_base import PreTokenizedInput, TextInput -from transformers.utils import TensorType - -MiniCPMOBatchFeature: TypeAlias = BatchFeature - - -class MiniCPMOProcessor(ProcessorMixin): - r"""Constructs a MiniCPMV processor which wraps a MiniCPMV image - processor and a MiniCPMV tokenizer into a single processor. - - [`MiniCPMVProcessor`] offers all the functionalities of - [`MiniCPMVImageProcessor`] and [`LlamaTokenizerWrapper`]. See the - [`~MiniCPMVProcessor.__call__`] and [`~MiniCPMVProcessor.decode`] - for more information. - - Args: - image_processor ([`MiniCPMVImageProcessor`], *optional*): - The image processor is a required input. - tokenizer ([`LlamaTokenizerWrapper`], *optional*): - The tokenizer is a required input. - - """ - - attributes = ["image_processor", "feature_extractor", "tokenizer"] - feature_extractor_class = "WhisperFeatureExtractor" - image_processor_class = "AutoImageProcessor" - tokenizer_class = "AutoTokenizer" - - def __init__( - self, - image_processor=None, - feature_extractor=None, - tokenizer=None, - pool_step=2, - ): - super().__init__(image_processor, feature_extractor, tokenizer) - # Mirror the MiniCPMVProcessor guard: newer (transformers v5.7+) - # MiniCPM image processors may drop the legacy `version` attribute, - # so fall back to None instead of hard-crashing. `version` only - # special-cases the 2.5 tokenization path; other values take the - # default branch. - self.version = getattr(image_processor, "version", None) - self.pool_step = pool_step - - def __call__( - self, - text: TextInput | PreTokenizedInput | list[TextInput] | list[PreTokenizedInput], - images: ImageInput = None, - audios: np.ndarray | list[np.ndarray] | list[list[np.ndarray]] = None, - audio_parts: list | None = None, - max_length: int | None = None, - do_pad: bool | None = True, - max_slice_nums: int | None = None, - use_image_id: bool = True, - chunk_input: bool = False, - return_tensors: str | TensorType | None = TensorType.PYTORCH, - sampling_rate: int | None = 16000, - **kwargs, - ) -> MiniCPMOBatchFeature: - if images is not None: - image_inputs = self.image_processor( - images, - do_pad=do_pad, - max_slice_nums=max_slice_nums, - return_tensors=return_tensors, - ) - else: - image_inputs = None - - if audios is not None: - audio_features, audio_feature_lens, audio_phs = self.audio_feature_extract( - audios, audio_parts, chunk_input, sampling_rate - ) - else: - audio_features, audio_feature_lens, audio_phs = [], [], [] - - model_inputs = self._convert_omni_to_inputs( - image_inputs, - audio_phs, - text, - max_slice_nums=max_slice_nums, - use_image_id=use_image_id, - max_length=max_length, - **kwargs, - ) - - model_inputs["audio_features"] = audio_features - model_inputs["audio_feature_lens"] = audio_feature_lens - - return MiniCPMOBatchFeature(data={**model_inputs}) - - def get_audio_placeholder(self, audio_lens, chunk_input, chunk_length): - pool_step = self.pool_step - feature_lens = math.ceil(audio_lens / self.feature_extractor.hop_length) - - feature_lens = (feature_lens - 1) // 2 + 1 - output_lens = (feature_lens - pool_step) // pool_step + 1 - - audio_start = getattr(self.tokenizer, "audio_start", "") - - if chunk_input: - fbank_feat_in_chunk = int(chunk_length * 100) - cnn_feat_in_chunk = (fbank_feat_in_chunk - 1) // 2 + 1 - audio_embeds_in_chunk = (cnn_feat_in_chunk - pool_step) // pool_step + 1 - num_audio_chunks = ( - output_lens + audio_embeds_in_chunk - 1 - ) // audio_embeds_in_chunk - - place_holders = "" - total_unk_len = 0 - for _ in range(num_audio_chunks): - unk_len = min(audio_embeds_in_chunk, output_lens - total_unk_len) - place_holders += audio_start + "" * unk_len + audio_end - total_unk_len += unk_len - audio_placeholder = place_holders - else: - audio_placeholder = audio_start + "" * output_lens + audio_end - - return audio_placeholder - - def audio_feature_extract( - self, - audios: np.ndarray | list[np.ndarray] | list[list[np.ndarray]], - audio_parts: list | None = None, - chunk_input: bool | None = False, - sampling_rate: int | None = None, - chunk_length: int | None = 1, - **kwargs, - ): - if isinstance(audios, np.ndarray): - audios_list = [[audios]] - elif isinstance(audios[0], np.ndarray): - audios_list = [audios] - else: - audios_list = audios - - if audio_parts is not None: - assert len(audio_parts) == len(audios_list) - for parts, audios in zip(audio_parts, audios_list): - assert len(parts) == len(audios) - - audio_feature_lens_list = [] - audio_ph_list = [] - - audio_features_all = [] - - # audio placeholder not dependent on audio_parts - for audios in audios_list: - if audios: - audio_ph_list.append( - [ - self.get_audio_placeholder(len(a), chunk_input, chunk_length) - for a in audios - ] - ) - else: - audio_ph_list.append([]) - - for idx, audios in enumerate(audios_list): - if audio_parts is not None: - # same audio part merge - audio_part = audio_parts[idx] - merge_audio = [] - cur_audio = [] - for aid, (part, audio) in enumerate(zip(audio_part, audios)): - if aid == 0 or audio_part[aid] == audio_part[aid - 1]: - cur_audio.append(audio) - else: - merge_audio.append(np.hstack(cur_audio)) - cur_audio = [audio] - if cur_audio: - merge_audio.append(np.hstack(cur_audio)) - - else: - merge_audio = audios - - audio_feature_lens = [] - - # If the audio exceeds 30 seconds, split it into chunks every 30 seconds. - final_merge_audio = [] - max_audio_inp_len = 30 * (sampling_rate or 16000) - for audio in merge_audio: - if len(audio) <= max_audio_inp_len: - final_merge_audio.append(audio) - else: - for i in range(math.ceil(len(audio) / max_audio_inp_len)): - final_merge_audio.append( - audio[i * max_audio_inp_len : (i + 1) * max_audio_inp_len] - ) - - if audios: - audio_inputs = self.feature_extractor( - final_merge_audio, - sampling_rate=sampling_rate, - return_attention_mask=True, - padding="max_length", - return_tensors="pt", - **kwargs, - ) - audio_feature = audio_inputs["input_features"] - actual_lens = audio_inputs["attention_mask"].sum(dim=1) - - for feat, lens in zip(audio_feature, actual_lens): - audio_features_all.append(feat[:, :lens]) - audio_feature_lens.append(lens) - - audio_feature_lens = torch.hstack(audio_feature_lens) - audio_feature_lens_list.append(audio_feature_lens) - else: - audio_feature_lens_list.append([]) - - if audio_features_all: - audio_features = [i.permute(1, 0) for i in audio_features_all] - audio_features = torch.nn.utils.rnn.pad_sequence( - audio_features, batch_first=True, padding_value=0.0 - ).permute(0, 2, 1) - else: - audio_features = [] - - return audio_features, audio_feature_lens_list, audio_ph_list - - # Copied from transformers.models.clip.processing_clip.CLIPProcessor.batch_decode - # with CLIP->Llama - def batch_decode(self, *args, **kwargs): - """This method forwards all its arguments to LlamaTokenizerFast's - [`~PythonBackend.batch_decode`]. Please refer to the - docstring of this method for more information. - """ - output_ids = args[0] - result_text = [] - for result in output_ids: - result = result[result != 0] - if len(result) > 0 and result[0] == self.tokenizer.bos_id: - result = result[1:] - if len(result) > 0 and result[-1] == self.tokenizer.eos_id: - result = result[:-1] - result_text.append( - self.tokenizer.decode(result, *args[1:], **kwargs).strip() - ) - return result_text - - # Copied from transformers.models.clip.processing_clip.CLIPProcessor.decode - # with CLIP->Llama - def decode(self, *args, **kwargs): - """This method forwards all its arguments to LlamaTokenizerFast's - [`~PythonBackend.decode`]. Please refer to the docstring - of this method for more information. - """ - result = args[0] - result = result[result != 0] - if len(result) > 0 and result[0] == self.tokenizer.bos_id: - result = result[1:] - if len(result) > 0 and ( - result[-1] == self.tokenizer.eos_id - or ( - hasattr(self.tokenizer, "eot_id") - and result[-1] == self.tokenizer.eot_id - ) - ): - result = result[:-1] - return self.tokenizer.decode(result, *args[1:], **kwargs).strip() - - def _convert(self, input_str, max_inp_length: int | None = None, **kwargs): - input_ids = self.tokenizer.encode(input_str, **kwargs) - if max_inp_length is not None: - input_ids = input_ids[:max_inp_length] - input_ids = torch.tensor(input_ids, dtype=torch.int32) - - ## image bound - start_cond = (input_ids == self.tokenizer.im_start_id) | ( - input_ids == self.tokenizer.slice_start_id - ) - end_cond = (input_ids == self.tokenizer.im_end_id) | ( - input_ids == self.tokenizer.slice_end_id - ) - - image_start_idx = torch.where(start_cond)[0] - image_start_idx += 1 - image_end_idx = torch.where(end_cond)[0] - - assert len(image_start_idx) == len(image_end_idx), ( - f"The number of image start tokens ({len(image_start_idx)}) " - f"and end tokens ({len(image_end_idx)}) must match." - ) - - image_bounds = torch.hstack( - [ - image_start_idx.unsqueeze(-1), - image_end_idx.unsqueeze(-1), - ] - ) - - ## audio bound - audio_start_idx = torch.where(input_ids == self.tokenizer.audio_start_id)[0] - audio_end_idx = torch.where(input_ids == self.tokenizer.audio_end_id)[0] - assert len(audio_start_idx) == len(audio_end_idx) - audio_bounds = torch.hstack( - [(audio_start_idx + 1).unsqueeze(-1), audio_end_idx.unsqueeze(-1)] - ) - - spk_start_idx = torch.where(input_ids == self.tokenizer.spk_start_id)[0] - spk_end_idx = torch.where(input_ids == self.tokenizer.spk_end_id)[0] - assert len(spk_start_idx) == len(spk_end_idx) - spk_bounds = torch.hstack( - [(spk_start_idx + 1).unsqueeze(-1), spk_end_idx.unsqueeze(-1)] - ) - - return input_ids, image_bounds, audio_bounds, spk_bounds - - def _convert_omni_to_inputs( - self, - images, - audio_phs, - texts: str | list[str], - truncation=None, - max_length=None, - max_slice_nums=None, - use_image_id=None, - return_tensors=None, - **kwargs, - ): - if images is None and audio_phs is None: - model_inputs = self.tokenizer( - texts, - return_tensors=return_tensors, - truncation=truncation, - max_length=max_length, - **kwargs, - ) - return MiniCPMOBatchFeature(data={**model_inputs}) - - image_tag = "(./)" - image_pattern = r"\(./\)" - audio_tag = "()" - audio_pattern = r"\(\)" - split_pattern = rf"({image_pattern}|{audio_pattern})" - - if isinstance(texts, str): - texts = [texts] - - bs = len(texts) - if images is not None: - images, image_sizes, tgt_sizes = ( - images["pixel_values"], - images["image_sizes"], - images["tgt_sizes"], - ) - else: - images, image_sizes, tgt_sizes = [[]] * bs, [[]] * bs, [[]] * bs - - input_ids_list = [] - image_bounds_list = [] - audio_bounds_list = [] - spk_bounds_list = [] - - for index, text in enumerate(texts): - text_chunks = regex.split(split_pattern, text) - - image_tags = regex.findall(image_pattern, text) - audio_tags = regex.findall(audio_pattern, text) - - if image_tags: - assert images is not None - assert len(image_tags) == len(image_sizes[index]) - if audio_tags: - assert audio_phs is not None - assert len(audio_tags) == len(audio_phs[index]) - - image_id = 0 - audio_id = 0 - for i, chunk in enumerate(text_chunks): - if chunk == image_tag: - image_placeholder = ( - self.image_processor.get_slice_image_placeholder( - image_sizes[index][image_id], - image_id, - max_slice_nums, - use_image_id, - ) - ) - image_id += 1 - text_chunks[i] = image_placeholder - elif chunk == audio_tag: - audio_placeholder = audio_phs[index][audio_id] - audio_id += 1 - text_chunks[i] = audio_placeholder - - final_text = "".join(text_chunks) - input_ids, image_bounds, audio_bounds, spk_bounds = self._convert( - final_text, max_length, **kwargs - ) - - input_ids_list.append(input_ids) - image_bounds_list.append(image_bounds) - audio_bounds_list.append(audio_bounds) - spk_bounds_list.append(spk_bounds) - - padded_input_ids, padding_lengths = self.pad( - input_ids_list, padding_side="left" - ) - attention_mask = torch.ones_like(padded_input_ids, dtype=torch.bool) - for i, length in enumerate(padding_lengths): - image_bounds_list[i] = image_bounds_list[i] + length - audio_bounds_list[i] = audio_bounds_list[i] + length - spk_bounds_list[i] = spk_bounds_list[i] + length - attention_mask[i, :length] = False - - data = { - "input_ids": padded_input_ids, - "attention_mask": attention_mask, - "pixel_values": images, - "image_sizes": image_sizes, - "image_bound": image_bounds_list, - "tgt_sizes": tgt_sizes, - "audio_bounds": audio_bounds_list, - "spk_bounds": spk_bounds_list, - } - - return data - - @property - # Copied from - # transformers.models.clip.processing_clip.CLIPProcessor.model_input_names - def model_input_names(self): - tokenizer_input_names = self.tokenizer.model_input_names - image_processor_input_names = self.image_processor.model_input_names - feature_extractor_input_names = self.feature_extractor.model_input_names - return list( - dict.fromkeys( - tokenizer_input_names - + image_processor_input_names - + feature_extractor_input_names - ) - ) - - def pad( - self, - inputs, - max_length=None, - padding_value=0, - padding_side="left", - ): - if not inputs: - return torch.empty(0), [] - - items = [] - if isinstance(inputs[0], list): - assert isinstance(inputs[0][0], torch.Tensor) - for it in inputs: - for tr in it: - items.append(tr) - else: - assert isinstance(inputs[0], torch.Tensor) - items = inputs - - batch_size = len(items) - shape = items[0].shape - dim = len(shape) - assert dim <= 2 - if max_length is None: - max_length = 0 - max_length = max(max_length, max(item.shape[-1] for item in items)) - min_length = min(item.shape[-1] for item in items) - dtype = items[0].dtype - - if dim == 0: - return torch.stack([item for item in items], dim=0), [0] - elif dim == 1: - if max_length == min_length: - return ( - torch.stack([item for item in items], dim=0), - [0] * batch_size, - ) - tensor = torch.zeros((batch_size, max_length), dtype=dtype) + padding_value - else: - tensor = ( - torch.zeros((batch_size, max_length, shape[-1]), dtype=dtype) - + padding_value - ) - - padding_length = [] - for i, item in enumerate(items): - if dim == 1: - if padding_side == "left": - tensor[i, -len(item) :] = item.clone() - else: - tensor[i, : len(item)] = item.clone() - elif dim == 2: - if padding_side == "left": - tensor[i, -len(item) :, :] = item.clone() - else: - tensor[i, : len(item), :] = item.clone() - padding_length.append(tensor.shape[-1] - len(item)) - - return tensor, padding_length diff --git a/vllm/transformers_utils/processors/minicpmv.py b/vllm/transformers_utils/processors/minicpmv.py deleted file mode 100644 index aaedaa3767..0000000000 --- a/vllm/transformers_utils/processors/minicpmv.py +++ /dev/null @@ -1,317 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -# Copyright 2024 The HuggingFace Inc. team. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Processor class for MiniCPMV.""" - -from typing import TypeAlias - -import regex -import torch -from transformers.image_processing_utils import BatchFeature -from transformers.image_utils import ImageInput -from transformers.processing_utils import ProcessorMixin -from transformers.tokenization_utils_base import ( - PaddingStrategy, - PreTokenizedInput, - TextInput, - TruncationStrategy, -) -from transformers.utils import TensorType - -MiniCPMVBatchFeature: TypeAlias = BatchFeature - - -class MiniCPMVProcessor(ProcessorMixin): - r"""Constructs a MiniCPMV processor which wraps a MiniCPMV image - processor and a MiniCPMV tokenizer into a single processor. - - [`MiniCPMVProcessor`] offers all the functionalities of - [`MiniCPMVImageProcessor`] and [`LlamaTokenizerWrapper`]. See the - [`~MiniCPMVProcessor.__call__`] and [`~MiniCPMVProcessor.decode`] - for more information. - - Args: - image_processor ([`MiniCPMVImageProcessor`], *optional*): - The image processor is a required input. - tokenizer ([`LlamaTokenizerWrapper`], *optional*): - The tokenizer is a required input. - - """ - - attributes = ["image_processor", "tokenizer"] - image_processor_class = "AutoImageProcessor" - tokenizer_class = "AutoTokenizer" - - def __init__(self, image_processor=None, tokenizer=None, version=None): - super().__init__(image_processor, tokenizer) - self.version = version - - def __call__( - self, - text: TextInput | PreTokenizedInput | list[TextInput] | list[PreTokenizedInput], - images: ImageInput = None, - padding: bool | str | PaddingStrategy = False, - truncation: bool | str | TruncationStrategy = None, - max_length: int | None = None, - do_pad: bool | None = True, - return_tensors: str | TensorType | None = TensorType.PYTORCH, - ) -> MiniCPMVBatchFeature: - """Run the vendored MiniCPMV processor on a (text, images) pair. - - Batched inputs are supported following the upstream MiniCPM-V - processor flow. ``images`` is forwarded to the underlying image - processor and ``text`` is tokenized with image placeholders - replaced by the appropriate slice tokens. Returns a - ``MiniCPMVBatchFeature`` with at minimum ``input_ids`` and (when - images are provided) ``pixel_values``, ``image_sizes``, - ``image_bound`` and ``tgt_sizes``. - """ - if images is not None: - image_inputs = self.image_processor( - images, do_pad=do_pad, return_tensors=return_tensors - ) - else: - image_inputs = {} - return self._convert_images_texts_to_inputs( - image_inputs, text, max_length=max_length - ) - - # Copied from transformers.models.clip.processing_clip.CLIPProcessor - # .batch_decode with CLIP->Llama - def batch_decode(self, *args, **kwargs): - """This method forwards all its arguments to LlamaTokenizerFast's - [`~PythonBackend.batch_decode`]. Please refer to the - docstring of this method for more information. - """ - output_ids = args[0] - result_text = [] - - bos_id = getattr( - self.tokenizer, - "bos_token_id", - getattr(self.tokenizer, "bos_id", 1), - ) - eos_id = getattr( - self.tokenizer, - "eos_token_id", - getattr(self.tokenizer, "eos_id", 2), - ) - - for result in output_ids: - result = result[result != 0] - if len(result) > 0 and result[0] == bos_id: - result = result[1:] - if len(result) > 0 and result[-1] == eos_id: - result = result[:-1] - result_text.append( - self.tokenizer.decode(result, *args[1:], **kwargs).strip() - ) - return result_text - - # Copied from transformers.models.clip.processing_clip.CLIPProcessor - # .decode with CLIP->Llama - def decode(self, *args, **kwargs): - """This method forwards all its arguments to LlamaTokenizerFast's - [`~PythonBackend.decode`]. Please refer to the docstring - of this method for more information. - """ - result = args[0] - result = result[result != 0] - - bos_id = getattr( - self.tokenizer, - "bos_token_id", - getattr(self.tokenizer, "bos_id", 1), - ) - eos_id = getattr( - self.tokenizer, - "eos_token_id", - getattr(self.tokenizer, "eos_id", 2), - ) - eot_id = getattr(self.tokenizer, "eot_id", None) - - if len(result) > 0 and result[0] == bos_id: - result = result[1:] - if len(result) > 0 and ( - result[-1] == eos_id or (eot_id is not None and result[-1] == eot_id) - ): - result = result[:-1] - return self.tokenizer.decode(result, *args[1:], **kwargs).strip() - - def _convert(self, input_str, max_inp_length: int | None = None): - add_bos = getattr(self.tokenizer, "add_bos_token", False) - if self.version == (2, 5) or add_bos: - input_ids = self.tokenizer.encode(input_str) - else: - bos_id = getattr( - self.tokenizer, - "bos_token_id", - getattr(self.tokenizer, "bos_id", 1), - ) - input_ids = [bos_id] + self.tokenizer.encode(input_str) - - if max_inp_length is not None: - input_ids = input_ids[:max_inp_length] - input_ids = torch.tensor(input_ids, dtype=torch.int32) - - im_start_id = getattr( - self.tokenizer, - "im_start_id", - self.tokenizer.convert_tokens_to_ids(""), - ) - im_end_id = getattr( - self.tokenizer, - "im_end_id", - self.tokenizer.convert_tokens_to_ids(""), - ) - - image_start_tokens = torch.where(input_ids == im_start_id)[0] - image_start_tokens += 1 - image_end_tokens = torch.where(input_ids == im_end_id)[0] - assert len(image_start_tokens) == len(image_end_tokens), ( - f"The number of image start tokens ({len(image_start_tokens)}) " - f"and end tokens ({len(image_end_tokens)}) must match." - ) - image_bounds = torch.hstack( - [ - image_start_tokens.unsqueeze(-1), - image_end_tokens.unsqueeze(-1), - ] - ) - return input_ids, image_bounds - - def _convert_images_texts_to_inputs( - self, - images, - texts, - do_pad=False, - truncation=None, - max_length=None, - return_tensors=None, - ): - if not len(images): - model_inputs = self.tokenizer( - texts, - return_tensors=return_tensors, - padding=do_pad, - truncation=truncation, - max_length=max_length, - ) - return MiniCPMVBatchFeature(data={**model_inputs}) - - pattern = "(./)" - images_val = images["pixel_values"] - image_sizes = images["image_sizes"] - tgt_sizes = images["tgt_sizes"] - - if isinstance(texts, str): - texts = [texts] - - input_ids_list = [] - image_bounds_list = [] - - for index, text in enumerate(texts): - image_tags = regex.findall(pattern, text) - assert len(image_tags) == len(image_sizes[index]) - text_chunks = text.split(pattern) - final_text = "" - for i in range(len(image_tags)): - placeholder = self.image_processor.get_slice_image_placeholder( - image_sizes[index][i] - ) - final_text = final_text + text_chunks[i] + placeholder - final_text += text_chunks[-1] - input_ids, image_bounds = self._convert(final_text, max_length) - input_ids_list.append(input_ids) - image_bounds_list.append(image_bounds) - - padded_input_ids, padding_lengths = self.pad( - input_ids_list, - padding_side="left", - ) - for i, length in enumerate(padding_lengths): - image_bounds_list[i] = image_bounds_list[i] + length - - return MiniCPMVBatchFeature( - data={ - "input_ids": padded_input_ids, - "attention_mask": padded_input_ids.ne(0), - "pixel_values": images_val, - "image_sizes": image_sizes, - "image_bound": image_bounds_list, - "tgt_sizes": tgt_sizes, - } - ) - - @property - # Copied from - # transformers.models.clip.processing_clip.CLIPProcessor.model_input_names - def model_input_names(self): - tokenizer_input_names = self.tokenizer.model_input_names - image_processor_input_names = self.image_processor.model_input_names - return list(dict.fromkeys(tokenizer_input_names + image_processor_input_names)) - - # Copied from openbmb/MiniCPM-V-4_5 processing_minicpmv.py. - def pad(self, inputs, max_length=None, padding_value=0, padding_side="left"): - if not inputs: - return torch.empty(0), [] - - items = [] - if isinstance(inputs[0], list): - assert isinstance(inputs[0][0], torch.Tensor) - for it in inputs: - for tr in it: - items.append(tr) - else: - assert isinstance(inputs[0], torch.Tensor) - items = inputs - - batch_size = len(items) - shape = items[0].shape - dim = len(shape) - assert dim <= 2 - if max_length is None: - max_length = 0 - max_length = max(max_length, max(item.shape[-1] for item in items)) - min_length = min(item.shape[-1] for item in items) - dtype = items[0].dtype - - if dim == 0: - return torch.stack([item for item in items], dim=0), [0] - elif dim == 1: - if max_length == min_length: - return torch.stack([item for item in items], dim=0), [0] * batch_size - tensor = torch.zeros((batch_size, max_length), dtype=dtype) + padding_value - else: - tensor = ( - torch.zeros((batch_size, max_length, shape[-1]), dtype=dtype) - + padding_value - ) - - padding_lengths = [] - for i, item in enumerate(items): - if dim == 1: - if padding_side == "left": - tensor[i, -len(item) :] = item.clone() - else: - tensor[i, : len(item)] = item.clone() - elif dim == 2: - if padding_side == "left": - tensor[i, -len(item) :, :] = item.clone() - else: - tensor[i, : len(item), :] = item.clone() - padding_lengths.append(tensor.shape[-1] - len(item)) - - return tensor, padding_lengths diff --git a/vllm/transformers_utils/processors/minimax_m3.py b/vllm/transformers_utils/processors/minimax_m3.py deleted file mode 100644 index 9d922f8e4c..0000000000 --- a/vllm/transformers_utils/processors/minimax_m3.py +++ /dev/null @@ -1,660 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""MiniMax M3 VL HuggingFace-compatible Processor / ImageProcessor / -VideoProcessor, vendored into vLLM so the model loads without -``--trust-remote-code`` (the released checkpoint only ships these classes as -remote code via ``auto_map``). - -Adapted verbatim from the ``MiniMaxAI/Minimax-M3-preview`` repository files -``image_processor.py``, ``video_processor.py`` and ``processing_minimax.py`` -(revision ``db01c0fe``). Both image and video processors use Qwen-style -``smart_resize`` (bound by total pixels). The original async frame-sampling -helpers are intentionally omitted: vLLM performs its own frame loading and -feeds decoded frames to the processor. -""" - -import math - -import torch -from torchvision.transforms import InterpolationMode -from transformers import AutoTokenizer, BatchFeature -from transformers.image_processing_backends import ( - TorchvisionBackend, - group_images_by_shape, - reorder_images, -) -from transformers.image_utils import PILImageResampling, SizeDict -from transformers.processing_utils import ( - ImagesKwargs, - ProcessingKwargs, - ProcessorMixin, - Unpack, - VideosKwargs, -) -from transformers.utils import TensorType -from transformers.video_processing_utils import BaseVideoProcessor -from transformers.video_utils import group_videos_by_shape, reorder_videos - -# Maximum allowed aspect ratio before smart_resize rejects the input. -MAX_RATIO = 200 - -# Fixed (non-configurable) bounds for the long-side resize logic, per the -# MiniMax-M3 size spec. ``min_short_side_pixel`` is the floor the short edge is -# enlarged to; ``*_MAX_TOTAL_PIXELS`` is the hard area cap that, once exceeded, -# aborts processing instead of downscaling. -MIN_SHORT_SIDE_PIXEL = 112 -IMAGE_MAX_TOTAL_PIXELS = 12_845_056 # 3584 ** 2 (width * height) -VIDEO_MAX_TOTAL_PIXELS = 301_056_000 # width * height * frames - - -def round_by_factor(number: int | float, factor: int) -> int: - return round(number / factor) * factor - - -def ceil_by_factor(number: int | float, factor: int) -> int: - return math.ceil(number / factor) * factor - - -def floor_by_factor(number: int | float, factor: int) -> int: - return math.floor(number / factor) * factor - - -def _smart_resize_by_long_side( - height: int, - width: int, - factor: int, - max_long_side_pixel: int, - min_short_side_pixel: int, - max_total_pixels: int | None, -) -> tuple[int, int]: - """Long-side based resize (MiniMax-M3 size spec). - - (a) if the long side exceeds ``max_long_side_pixel`` → shrink so the long - side equals ``max_long_side_pixel``; - (b) else if the short side is below ``min_short_side_pixel`` → enlarge so the - short side equals ``min_short_side_pixel``; - (c) if the resulting area still exceeds ``max_total_pixels`` → raise. - - (a) and (b) are mutually exclusive (they branch on the *original* long side). - Both sides are then rounded to a multiple of ``factor``. For videos the - ``max_total_pixels`` cap is volumetric (width * height * frames) and is - enforced by the caller, so pass ``max_total_pixels=None`` here. - """ - long_side = max(height, width) - short_side = min(height, width) - - scaled_height: float = height - scaled_width: float = width - if long_side > max_long_side_pixel: - beta = max_long_side_pixel / long_side - scaled_height = height * beta - scaled_width = width * beta - elif short_side < min_short_side_pixel: - beta = min_short_side_pixel / short_side - scaled_height = height * beta - scaled_width = width * beta - - h_bar = max(factor, round_by_factor(scaled_height, factor)) - w_bar = max(factor, round_by_factor(scaled_width, factor)) - - if max_total_pixels is not None and h_bar * w_bar > max_total_pixels: - raise ValueError( - f"image area {h_bar * w_bar} exceeds max_total_pixels " - f"{max_total_pixels} after resizing" - ) - return h_bar, w_bar - - -def smart_resize( - height: int, - width: int, - factor: int = 28, - min_pixels: int = 4 * 28 * 28, - max_pixels: int = 451584, - max_long_side_pixel: int | None = None, - min_short_side_pixel: int = MIN_SHORT_SIDE_PIXEL, - max_total_pixels: int | None = None, -) -> tuple[int, int]: - """Rescale (height, width) so each side is a multiple of ``factor``. - - When ``max_long_side_pixel`` is set, use the MiniMax-M3 long-side resize - spec (see :func:`_smart_resize_by_long_side`). Otherwise fall back to the - Qwen-VL area bound, keeping the total area within ``[min_pixels, max_pixels]``. - """ - if max(height, width) / min(height, width) > MAX_RATIO: - raise ValueError( - f"absolute aspect ratio must be smaller than {MAX_RATIO}, " - f"got {max(height, width) / min(height, width)}" - ) - if max_long_side_pixel is not None: - return _smart_resize_by_long_side( - height, - width, - factor=factor, - max_long_side_pixel=max_long_side_pixel, - min_short_side_pixel=min_short_side_pixel, - max_total_pixels=max_total_pixels, - ) - h_bar = max(factor, round_by_factor(height, factor)) - w_bar = max(factor, round_by_factor(width, factor)) - if h_bar * w_bar > max_pixels: - beta = math.sqrt((height * width) / max_pixels) - h_bar = floor_by_factor(height / beta, factor) - w_bar = floor_by_factor(width / beta, factor) - elif h_bar * w_bar < min_pixels: - beta = math.sqrt(min_pixels / (height * width)) - h_bar = ceil_by_factor(height * beta, factor) - w_bar = ceil_by_factor(width * beta, factor) - return h_bar, w_bar - - -class MiniMaxM3VLImageProcessorKwargs(ImagesKwargs, total=False): # type: ignore[call-arg] - patch_size: int - temporal_patch_size: int - merge_size: int - max_pixels: int - max_long_side_pixel: int - - -class MiniMaxM3VLImageProcessor(TorchvisionBackend): - do_resize = True - resample = PILImageResampling.BICUBIC - # required by base-class validation, not used as the resize bound - size = {"height": 672, "width": 672} - default_to_square = False - do_rescale = True - rescale_factor = 1 / 255 - do_normalize = True - image_mean = [0.48145466, 0.4578275, 0.40821073] - image_std = [0.26862954, 0.26130258, 0.27577711] - do_convert_rgb = True - patch_size = 14 - temporal_patch_size = 2 - merge_size = 2 - max_pixels = 451584 # 672 * 672 - # Long-side resize spec (opt-in via ``max_long_side_pixel``). The latter two - # are fixed per the spec and are not exposed as configurable kwargs. - max_long_side_pixel = None - min_short_side_pixel = MIN_SHORT_SIDE_PIXEL - max_total_pixels = IMAGE_MAX_TOTAL_PIXELS - valid_kwargs = MiniMaxM3VLImageProcessorKwargs - model_input_names = ["pixel_values", "image_grid_thw"] - - def __init__(self, **kwargs: Unpack[MiniMaxM3VLImageProcessorKwargs]): - super().__init__(**kwargs) - - def preprocess( - self, images, **kwargs: Unpack[MiniMaxM3VLImageProcessorKwargs] - ) -> BatchFeature: - return super().preprocess(images, **kwargs) - - def _preprocess( - self, - images: list[torch.Tensor], - do_resize: bool, - size: SizeDict, - resample: "PILImageResampling | InterpolationMode | int | None", - do_rescale: bool, - rescale_factor: float, - do_normalize: bool, - image_mean: "float | list[float] | None", - image_std: "float | list[float] | None", - patch_size: int, - temporal_patch_size: int, - merge_size: int, - max_pixels: int, - max_long_side_pixel: "int | None", - disable_grouping: "bool | None", - return_tensors: "str | TensorType | None", - **kwargs, - ) -> BatchFeature: - grouped_images, grouped_images_index = group_images_by_shape( - images, disable_grouping=disable_grouping - ) - resized_images_grouped = {} - factor = patch_size * merge_size - for shape, stacked_images in grouped_images.items(): - height, width = stacked_images.shape[-2:] - if do_resize: - resized_height, resized_width = smart_resize( - height, - width, - factor=factor, - max_pixels=max_pixels, - max_long_side_pixel=max_long_side_pixel, - min_short_side_pixel=self.min_short_side_pixel, - max_total_pixels=self.max_total_pixels, - ) - stacked_images = self.resize( - stacked_images, - size=SizeDict(height=resized_height, width=resized_width), - resample=resample, - ) - resized_images_grouped[shape] = stacked_images - - resized_images = reorder_images(resized_images_grouped, grouped_images_index) - - grouped_images, grouped_images_index = group_images_by_shape( - resized_images, disable_grouping=disable_grouping - ) - processed_images_grouped = {} - processed_grids = {} - - for shape, stacked_images in grouped_images.items(): - resized_height, resized_width = stacked_images.shape[-2:] - - patches = self.rescale_and_normalize( - stacked_images, - do_rescale, - rescale_factor, - do_normalize, - image_mean, - image_std, - ) - if patches.ndim == 4: - patches = patches.unsqueeze(1) - - if patches.shape[1] % temporal_patch_size != 0: - repeats = patches[:, -1:].repeat( - 1, - temporal_patch_size - (patches.shape[1] % temporal_patch_size), - 1, - 1, - 1, - ) - patches = torch.cat([patches, repeats], dim=1) - - batch_size, grid_t, channel = patches.shape[:3] - grid_t = grid_t // temporal_patch_size - grid_h, grid_w = resized_height // patch_size, resized_width // patch_size - - patches = patches.view( - batch_size, - grid_t, - temporal_patch_size, - channel, - grid_h // merge_size, - merge_size, - patch_size, - grid_w // merge_size, - merge_size, - patch_size, - ) - patches = patches.permute(0, 1, 4, 7, 5, 8, 3, 2, 6, 9) - - flatten_patches = patches.reshape( - batch_size, - grid_t * grid_h * grid_w, - channel * temporal_patch_size * patch_size * patch_size, - ) - - processed_images_grouped[shape] = flatten_patches - processed_grids[shape] = [[grid_t, grid_h, grid_w]] * batch_size - - processed_images = reorder_images( - processed_images_grouped, grouped_images_index - ) - processed_grids = reorder_images(processed_grids, grouped_images_index) - - pixel_values = torch.cat(processed_images, dim=0) - image_grid_thw = torch.tensor(processed_grids, dtype=torch.long) - - return BatchFeature( - data={"pixel_values": pixel_values, "image_grid_thw": image_grid_thw}, - tensor_type=return_tensors, - ) - - def get_number_of_image_patches(self, height: int, width: int, images_kwargs=None): - images_kwargs = images_kwargs or {} - patch_size = images_kwargs.get("patch_size", self.patch_size) - merge_size = images_kwargs.get("merge_size", self.merge_size) - max_pixels = images_kwargs.get("max_pixels", self.max_pixels) - max_long_side_pixel = images_kwargs.get( - "max_long_side_pixel", self.max_long_side_pixel - ) - - resized_height, resized_width = smart_resize( - height, - width, - factor=patch_size * merge_size, - max_pixels=max_pixels, - max_long_side_pixel=max_long_side_pixel, - min_short_side_pixel=self.min_short_side_pixel, - max_total_pixels=self.max_total_pixels, - ) - grid_h, grid_w = resized_height // patch_size, resized_width // patch_size - return grid_h * grid_w - - -class MiniMaxM3VLVideoProcessorKwargs(VideosKwargs, total=False): # type: ignore[call-arg] - patch_size: int - temporal_patch_size: int - merge_size: int - min_pixels: int - max_pixels: int - max_long_side_pixel: int - total_pixels: int - min_frames: int - max_frames: int - fps: "float | int" - - -class MiniMaxM3VLVideoProcessor(BaseVideoProcessor): - do_resize = True - resample = PILImageResampling.BICUBIC - size = {"height": 672, "width": 672} - default_to_square = False - do_rescale = True - rescale_factor = 1 / 255 - do_normalize = True - image_mean = [0.48145466, 0.4578275, 0.40821073] - image_std = [0.26862954, 0.26130258, 0.27577711] - do_convert_rgb = True - do_sample_frames = False - patch_size = 14 - temporal_patch_size = 2 - merge_size = 2 - min_pixels = 4 * 28 * 28 - max_pixels = 768 * 28 * 28 # 602,112 - total_pixels = int(64000 * 28 * 28 * 0.9) # ~45M, ~64k tokens budget - # Long-side resize spec (opt-in via ``max_long_side_pixel``). The video - # ``max_total_pixels`` cap is volumetric (width * height * frames) and is - # enforced in ``_preprocess`` once the frame count is known. - max_long_side_pixel = None - min_short_side_pixel = MIN_SHORT_SIDE_PIXEL - max_total_pixels = VIDEO_MAX_TOTAL_PIXELS - fps = 1.0 - min_frames = 4 - max_frames = 768 - valid_kwargs = MiniMaxM3VLVideoProcessorKwargs - model_input_names = ["pixel_values_videos", "video_grid_thw"] - - def __init__(self, **kwargs: Unpack[MiniMaxM3VLVideoProcessorKwargs]): - super().__init__(**kwargs) - - def _preprocess( - self, - videos: list[torch.Tensor], - do_resize: bool, - size: SizeDict, - resample: "PILImageResampling | InterpolationMode | int | None", - do_rescale: bool, - rescale_factor: float, - do_normalize: bool, - image_mean: "float | list[float] | None", - image_std: "float | list[float] | None", - patch_size: int, - temporal_patch_size: int, - merge_size: int, - min_pixels: int, - max_pixels: int, - max_long_side_pixel: "int | None" = None, - return_tensors: "str | TensorType | None" = None, - **kwargs, - ) -> BatchFeature: - grouped_videos, grouped_videos_index = group_videos_by_shape(videos) - resized_videos_grouped = {} - factor = patch_size * merge_size - for shape, stacked_videos in grouped_videos.items(): - batch_size, num_frames, channels, height, width = stacked_videos.shape - resized_height, resized_width = height, width - if do_resize: - resized_height, resized_width = smart_resize( - height, - width, - factor=factor, - min_pixels=min_pixels, - max_pixels=max_pixels, - max_long_side_pixel=max_long_side_pixel, - min_short_side_pixel=self.min_short_side_pixel, - # Per-frame raise disabled; the video cap is volumetric and - # is enforced below once num_frames is known. - max_total_pixels=None, - ) - if ( - max_long_side_pixel is not None - and resized_height * resized_width * num_frames - > self.max_total_pixels - ): - raise ValueError( - f"video area {resized_height * resized_width * num_frames} " - f"(width * height * frames) exceeds max_total_pixels " - f"{self.max_total_pixels} after resizing" - ) - stacked_videos = stacked_videos.view( - batch_size * num_frames, channels, height, width - ) - stacked_videos = self.resize( - stacked_videos, - size=SizeDict(height=resized_height, width=resized_width), - resample=resample, - ) - stacked_videos = stacked_videos.view( - batch_size, - num_frames, - channels, - resized_height, - resized_width, - ) - resized_videos_grouped[shape] = stacked_videos - resized_videos = reorder_videos(resized_videos_grouped, grouped_videos_index) - - grouped_videos, grouped_videos_index = group_videos_by_shape(resized_videos) - processed_videos_grouped = {} - processed_grids = {} - for shape, stacked_videos in grouped_videos.items(): - resized_height, resized_width = stacked_videos.shape[-2:] - patches = self.rescale_and_normalize( - stacked_videos, - do_rescale, - rescale_factor, - do_normalize, - image_mean, - image_std, - ) - - if pad := -patches.shape[1] % temporal_patch_size: - repeats = patches[:, -1:].expand(-1, pad, -1, -1, -1) - patches = torch.cat([patches, repeats], dim=1) - - batch_size, grid_t, channels = patches.shape[:3] - grid_t = grid_t // temporal_patch_size - grid_h, grid_w = resized_height // patch_size, resized_width // patch_size - - patches = patches.view( - batch_size, - grid_t, - temporal_patch_size, - channels, - grid_h // merge_size, - merge_size, - patch_size, - grid_w // merge_size, - merge_size, - patch_size, - ) - patches = patches.permute(0, 1, 4, 7, 5, 8, 3, 2, 6, 9) - flatten_patches = patches.reshape( - batch_size, - grid_t * grid_h * grid_w, - channels * temporal_patch_size * patch_size * patch_size, - ) - - processed_videos_grouped[shape] = flatten_patches - processed_grids[shape] = [[grid_t, grid_h, grid_w]] * batch_size - - processed_videos = reorder_videos( - processed_videos_grouped, grouped_videos_index - ) - processed_grids = reorder_videos(processed_grids, grouped_videos_index) - pixel_values_videos = torch.cat(processed_videos, dim=0) - video_grid_thw = torch.tensor(processed_grids, dtype=torch.long) - - return BatchFeature( - data={ - "pixel_values_videos": pixel_values_videos, - "video_grid_thw": video_grid_thw, - }, - tensor_type=return_tensors, - ) - - -class MiniMaxVLProcessorKwargs(ProcessingKwargs, total=False): # type: ignore[call-arg] - _defaults = { - "videos_kwargs": { - "do_resize": False, - "return_metadata": True, - }, - } - - -class MiniMaxVLProcessor(ProcessorMixin): - IMAGE_TOKEN = "]<]image[>[" - VIDEO_TOKEN = "]<]video[>[" - VISION_START_TOKEN = "]<]start of image[>[" - VISION_END_TOKEN = "]<]end of image[>[" - - @classmethod - def from_pretrained(cls, pretrained_model_name_or_path, **kwargs): - # Bypass ProcessorMixin's dynamic module lookup, which breaks in - # transformers >= 5.9 when image_processor_class is a string: the - # register() API now stores classes as {"pil": cls} dicts in - # _extra_content, but get_possibly_dynamic_module() still calls - # .__name__ on the raw value, crashing with AttributeError on dicts. - # - # Reuse the tokenizer passed by vLLM instead of loading another one, - # and keep it out of the kwargs forwarded to the sub-processor loaders. - tokenizer = kwargs.pop("tokenizer", None) - if tokenizer is None: - tokenizer = AutoTokenizer.from_pretrained( - pretrained_model_name_or_path, **kwargs - ) - image_processor = MiniMaxM3VLImageProcessor.from_pretrained( - pretrained_model_name_or_path, **kwargs - ) - video_processor = MiniMaxM3VLVideoProcessor.from_pretrained( - pretrained_model_name_or_path, **kwargs - ) - return cls( - image_processor=image_processor, - tokenizer=tokenizer, - video_processor=video_processor, - ) - - def __init__( - self, image_processor=None, tokenizer=None, video_processor=None, **kwargs - ): - self.image_token_id = tokenizer.convert_tokens_to_ids(self.IMAGE_TOKEN) - self.video_token_id = tokenizer.convert_tokens_to_ids(self.VIDEO_TOKEN) - super().__init__(image_processor, tokenizer, video_processor) - # Video expansion also uses image start/end tokens. Separate video - # start/end tokens exist in the tokenizer, but the original MiniMax - # serving path did not use them; keep that behavior for compatibility. - self.vision_start_token_id = tokenizer.convert_tokens_to_ids( - self.VISION_START_TOKEN - ) - self.vision_end_token_id = tokenizer.convert_tokens_to_ids( - self.VISION_END_TOKEN - ) - - def __call__( - self, - images=None, - text=None, - videos=None, - **kwargs: Unpack[MiniMaxVLProcessorKwargs], - ) -> BatchFeature: - output_kwargs = self._merge_kwargs( - MiniMaxVLProcessorKwargs, - tokenizer_init_kwargs=self.tokenizer.init_kwargs, - **kwargs, - ) - - if images is not None: - images_kwargs = output_kwargs["images_kwargs"] - image_inputs = self.image_processor(images=images, **images_kwargs) - image_grid_thw = image_inputs["image_grid_thw"] - else: - image_inputs = {} - image_grid_thw = None - - if videos is not None: - videos_kwargs = output_kwargs["videos_kwargs"] - video_inputs = self.video_processor(videos=videos, **videos_kwargs) - video_grid_thw = video_inputs["video_grid_thw"] - if not kwargs.get("return_metadata"): - video_metadata = video_inputs.pop("video_metadata") - else: - video_metadata = video_inputs["video_metadata"] - else: - video_inputs = {} - video_grid_thw = None - - if not isinstance(text, list): - text = [text] - text = text.copy() - - # Expand image tokens - if image_grid_thw is not None: - merge_length = self.image_processor.merge_size**2 - placeholder = "]<]placeholder[>[" - index = 0 - for i in range(len(text)): - while self.IMAGE_TOKEN in text[i]: - num_tokens = image_grid_thw[index].prod() // merge_length - text[i] = text[i].replace( - self.IMAGE_TOKEN, - self.VISION_START_TOKEN - + placeholder * num_tokens - + self.VISION_END_TOKEN, - 1, - ) - index += 1 - text[i] = text[i].replace(placeholder, self.IMAGE_TOKEN) - - # Expand video tokens - if video_grid_thw is not None: - merge_length = self.image_processor.merge_size**2 - placeholder = "]<]placeholder[>[" - index = 0 - for i in range(len(text)): - while self.VIDEO_TOKEN in text[i]: - metadata = video_metadata[index] - grid_t = video_grid_thw[index][0] - frame_seqlen = video_grid_thw[index][1:].prod() // merge_length - - video_placeholder = "" - for frame_idx in range(grid_t): - if ( - metadata.fps is not None - and metadata.frames_indices is not None - ): - ts = ( - metadata.frames_indices[ - min( - frame_idx - * self.video_processor.temporal_patch_size, - len(metadata.frames_indices) - 1, - ) - ] - / metadata.fps - ) - video_placeholder += f"]<]{ts:.1f} seconds[>[" - video_placeholder += ( - self.VISION_START_TOKEN - + placeholder * frame_seqlen - + self.VISION_END_TOKEN - ) - - text[i] = text[i].replace(self.VIDEO_TOKEN, video_placeholder, 1) - index += 1 - text[i] = text[i].replace(placeholder, self.VIDEO_TOKEN) - - # Tokenize - return_tensors = output_kwargs["text_kwargs"].pop("return_tensors", None) - text_inputs = self.tokenizer(text, **output_kwargs["text_kwargs"]) - - return BatchFeature( - data={**text_inputs, **image_inputs, **video_inputs}, - tensor_type=return_tensors, - ) diff --git a/vllm/transformers_utils/processors/moondream3.py b/vllm/transformers_utils/processors/moondream3.py deleted file mode 100644 index 321fc95171..0000000000 --- a/vllm/transformers_utils/processors/moondream3.py +++ /dev/null @@ -1,539 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Custom processor for Moondream3 model.""" - -import math - -import numpy as np -import torch -from PIL import Image -from transformers import AutoProcessor, BatchFeature -from transformers.image_utils import ImageInput -from transformers.processing_utils import ProcessingKwargs, ProcessorMixin, Unpack -from transformers.tokenization_utils_base import ( - PreTokenizedInput, - PreTrainedTokenizerBase, - TextInput, -) - -from vllm.multimodal.image import convert_image_mode - -__all__ = ["Moondream3Processor"] - - -class Moondream3ProcessorKwargs(ProcessingKwargs, total=False): # type: ignore[call-arg] - _defaults = { - "text_kwargs": { - "padding": False, - }, - "images_kwargs": { - "max_crops": 12, - "overlap_margin": 4, - "crop_size": 378, - "patch_size": 14, - "convert_to_rgb": True, - "return_tensors": "pt", - }, - } - - -def select_tiling( - height: int, width: int, crop_size: int, max_crops: int -) -> tuple[int, int]: - """Determine the optimal number of tiles to cover an image.""" - if height <= crop_size or width <= crop_size: - return (1, 1) - - min_h = math.ceil(height / crop_size) - min_w = math.ceil(width / crop_size) - - if min_h * min_w > max_crops: - ratio = math.sqrt(max_crops / (min_h * min_w)) - return (max(1, math.floor(min_h * ratio)), max(1, math.floor(min_w * ratio))) - - h_tiles = math.floor(math.sqrt(max_crops * height / width)) - w_tiles = math.floor(math.sqrt(max_crops * width / height)) - - h_tiles = max(h_tiles, min_h) - w_tiles = max(w_tiles, min_w) - - if h_tiles * w_tiles > max_crops: - if w_tiles > h_tiles: - w_tiles = math.floor(max_crops / h_tiles) - else: - h_tiles = math.floor(max_crops / w_tiles) - - return (max(1, h_tiles), max(1, w_tiles)) - - -class Moondream3Processor(ProcessorMixin): - """Constructs a Moondream3 processor which handles image preprocessing - and tokenization for the Moondream3 multimodal model. - - Args: - tokenizer: The tokenizer to use for text processing. - chat_template: Optional chat template string. - crop_size: Size of each image crop. - max_crops: Maximum number of crops per image. - overlap_margin: Margin for overlapping crops in patches. - patch_size: Size of each patch. - - """ - - attributes = ["tokenizer"] - valid_kwargs = [ - "chat_template", - "crop_size", - "max_crops", - "overlap_margin", - "patch_size", - ] - - tokenizer_class = "AutoTokenizer" - # Use separate tokenizer repo - _tokenizer_repo = "moondream/starmie-v1" - - # Default chat template for Moondream3 - # Moondream uses special tokens for prompting: - # - Token 0 (<|endoftext|>): BOS token (ALWAYS present at position 0) - # - Token 1 (<|md_reserved_0|>): Start of instruction - # - Token 2 (<|md_reserved_1|>): Separator before question - # - Token 3 (<|md_reserved_2|>): End of question / start of answer - # - # Task routing based on text prefix: - # "caption [short|normal|long]" → describe<|md_reserved_1|>{length} - # "describe [short|normal|long]" → describe<|md_reserved_1|>{length} - # otherwise → query<|md_reserved_1|> - # - # Format with image: - # <|endoftext|><|md_reserved_0|>{task}<|md_reserved_1|>{q}<|md_reserved_2|> - # Format without image: - # <|endoftext|><|md_reserved_0|>{task}<|md_reserved_1|>{q}<|md_reserved_2|> - _default_chat_template = ( - "{% for message in messages %}" - "{% if message['role'] == 'user' %}" - "{% if message['content'] is string %}" - # Simple string content (with image assumed) - route by prefix - "<|endoftext|><|md_reserved_0|>" - "{% if message['content'] == 'caption' %}" - "describe<|md_reserved_1|>normal<|md_reserved_2|>" - "{% elif message['content'].startswith('caption ') %}" - "describe<|md_reserved_1|>{{ message['content'][8:] }}<|md_reserved_2|>" - "{% elif message['content'] == 'describe' %}" - "describe<|md_reserved_1|>normal<|md_reserved_2|>" - "{% elif message['content'].startswith('describe ') %}" - "describe<|md_reserved_1|>{{ message['content'][9:] }}<|md_reserved_2|>" - "{% else %}" - "query<|md_reserved_1|>{{ message['content'] }}<|md_reserved_2|>" - "{% endif %}" - "{% else %}" - # List content - build Moondream's image prefix independently of - # OpenAI-style content part order, then render the text task. - "<|endoftext|>" - "{% for content in message['content'] %}" - "{% if content['type'] in ['image', 'image_url', 'input_image', 'image_pil'] %}" # noqa: E501 - "" - "{% endif %}" - "{% endfor %}" - "{% for content in message['content'] %}" - "{% if content['type'] == 'text' %}" - "<|md_reserved_0|>" - "{% if content['text'] == 'caption' %}" - "describe<|md_reserved_1|>normal<|md_reserved_2|>" - "{% elif content['text'].startswith('caption ') %}" - "describe<|md_reserved_1|>{{ content['text'][8:] }}<|md_reserved_2|>" - "{% elif content['text'] == 'describe' %}" - "describe<|md_reserved_1|>normal<|md_reserved_2|>" - "{% elif content['text'].startswith('describe ') %}" - "describe<|md_reserved_1|>{{ content['text'][9:] }}<|md_reserved_2|>" - "{% else %}" - "query<|md_reserved_1|>{{ content['text'] }}<|md_reserved_2|>" - "{% endif %}" - "{% endif %}" - "{% endfor %}" - "{% endif %}" - "{% elif message['role'] == 'assistant' %}" - "{{ message['content'] }}" - "{% endif %}" - "{% endfor %}" - ) - - def __init__( - self, - tokenizer: PreTrainedTokenizerBase | None = None, - chat_template: str | None = None, - crop_size: int = 378, - max_crops: int = 12, - overlap_margin: int = 4, - patch_size: int = 14, - **kwargs, - ): - self.image_token = "" - self.crop_size = crop_size - self.max_crops = max_crops - self.overlap_margin = overlap_margin - self.patch_size = patch_size - - # Number of patches per crop (27x27 = 729 for 378/14) - self.patches_per_crop = (crop_size // patch_size) ** 2 - - # Use default chat template if none provided - if chat_template is None: - chat_template = self._default_chat_template - - super().__init__(tokenizer, chat_template=chat_template) - - @classmethod - def from_pretrained( - cls, - pretrained_model_name_or_path, - **kwargs, - ): - """Load the processor, using a separate tokenizer repo. - - The moondream3 model uses a custom tokenizer from 'moondream/starmie-v1' - instead of having tokenizer files in the model repo. - """ - from transformers import AutoTokenizer, TokenizersBackend - from transformers.utils import cached_file - - tokenizer = kwargs.pop("tokenizer", None) - - tokenizer_kwargs = { - "trust_remote_code": kwargs.get("trust_remote_code", False), - } - for key in ( - "cache_dir", - "force_download", - "local_files_only", - "revision", - "subfolder", - "token", - ): - if key in kwargs: - tokenizer_kwargs[key] = kwargs[key] - - cached_file_kwargs = { - key: tokenizer_kwargs[key] - for key in ( - "cache_dir", - "force_download", - "local_files_only", - "revision", - "subfolder", - "token", - ) - if key in tokenizer_kwargs - } - - def load_tokenizer(repo_or_path): - try: - return AutoTokenizer.from_pretrained(repo_or_path, **tokenizer_kwargs) - except Exception: - tokenizer_file = cached_file( - repo_or_path, - "tokenizer.json", - **cached_file_kwargs, - ) - return TokenizersBackend( - tokenizer_file=tokenizer_file, - clean_up_tokenization_spaces=False, - ) - - if isinstance(tokenizer, str): - tokenizer = load_tokenizer(tokenizer) - - if tokenizer is None: - # Prefer model-local tokenizer files first. If unavailable, fall - # back to moondream's dedicated tokenizer repository. - try: - tokenizer = load_tokenizer(pretrained_model_name_or_path) - except Exception: - tokenizer = load_tokenizer(cls._tokenizer_repo) - - # Configure special tokens for Moondream3 - # BOS and EOS are both token 0 (<|endoftext|>), matching the native - # config (TokenizerConfig.bos_id=0, eos_id=0). This is standard for - # GPT-2 style models where <|endoftext|> signals both start and end. - # Token 1 (<|md_reserved_0|>) is a template delimiter, NOT the EOS. - tokenizer.bos_token = "<|endoftext|>" - tokenizer.bos_token_id = 0 - tokenizer.eos_token = "<|endoftext|>" - tokenizer.eos_token_id = 0 - - # Extract processor-specific kwargs - crop_size = kwargs.pop("crop_size", 378) - max_crops = kwargs.pop("max_crops", 12) - overlap_margin = kwargs.pop("overlap_margin", 4) - patch_size = kwargs.pop("patch_size", 14) - chat_template = kwargs.pop("chat_template", None) - - # Set default chat template on tokenizer if not already set - if chat_template is None: - chat_template = cls._default_chat_template - if tokenizer.chat_template is None: - tokenizer.chat_template = chat_template - - return cls( - tokenizer=tokenizer, - chat_template=chat_template, - crop_size=crop_size, - max_crops=max_crops, - overlap_margin=overlap_margin, - patch_size=patch_size, - ) - - def __call__( - self, - images: ImageInput = None, - text: TextInput - | PreTokenizedInput - | list[TextInput] - | list[PreTokenizedInput] = None, - **kwargs: Unpack[Moondream3ProcessorKwargs], - ) -> BatchFeature: - """Process images and text for Moondream3 model. - - Args: - images: Input images (PIL Image, numpy array, or list thereof). - text: Input text or list of texts. - **kwargs: Additional processing arguments. - - Returns: - BatchFeature with processed inputs. - - """ - output_kwargs = self._merge_kwargs( - Moondream3ProcessorKwargs, - tokenizer_init_kwargs=self.tokenizer.init_kwargs, - **kwargs, - ) - - # Process images - image_features = {} - if images is not None: - processed_images = [] - tilings = [] - - images_list = images if isinstance(images, list) else [images] - for image in images_list: - pixel_values, tiling = self.preprocess_image( - image, **output_kwargs["images_kwargs"] - ) - processed_images.append(pixel_values) - tilings.append(tiling) - - if processed_images: - image_features["pixel_values"] = processed_images - image_features["tilings"] = tilings - - # Process text - if text is not None: - if not isinstance(text, list): - text = [text] - - # Get text kwargs, remove keys we set ourselves - text_kwargs = output_kwargs.get("text_kwargs", {}).copy() - text_kwargs.pop("return_tensors", None) - text_kwargs.pop("add_special_tokens", None) - - # Tokenize text - tokenized = self.tokenizer( - text, - add_special_tokens=True, - return_tensors="pt", - **text_kwargs, - ) - - output = BatchFeature(data=dict(tokenized)) - - # Add image features - if image_features: - output["pixel_values"] = image_features["pixel_values"] - output["tilings"] = image_features["tilings"] - - return output - - # If only images were provided - return BatchFeature(data=image_features) - - @staticmethod - def _image_array_to_uint8(array: np.ndarray) -> np.ndarray: - if array.dtype == np.uint8: - return np.ascontiguousarray(array) - - if array.dtype == np.bool_: - return np.ascontiguousarray(array.astype(np.uint8) * 255) - - if np.issubdtype(array.dtype, np.floating): - array = np.nan_to_num(array, nan=0.0, posinf=255.0, neginf=0.0) - if array.size > 0 and array.max() <= 1.0: - array = array * 255.0 - array = np.rint(array) - - return np.ascontiguousarray(np.clip(array, 0, 255).astype(np.uint8)) - - @staticmethod - def _to_pil_image(image: ImageInput) -> Image.Image: - if isinstance(image, Image.Image): - return image - - if isinstance(image, torch.Tensor): - tensor = image.detach().cpu() - if tensor.dtype == torch.bfloat16: - tensor = tensor.to(torch.float32) - image_array = tensor.numpy() - elif isinstance(image, np.ndarray): - image_array = image - else: - raise TypeError( - "Moondream3 images must be PIL images, numpy arrays, " - f"or torch tensors, got {type(image)!r}." - ) - - if image_array.ndim == 2: - image_array = Moondream3Processor._image_array_to_uint8(image_array) - return Image.fromarray(image_array) - - if image_array.ndim != 3: - raise ValueError( - "Moondream3 image arrays must have 2 or 3 dimensions, " - f"got shape {image_array.shape}." - ) - - channel_dims = (1, 3, 4) - if image_array.shape[-1] not in channel_dims: - if image_array.shape[0] not in channel_dims: - raise ValueError( - "Moondream3 image arrays must be HWC or CHW with 1, 3, " - f"or 4 channels, got shape {image_array.shape}." - ) - image_array = np.transpose(image_array, (1, 2, 0)) - - image_array = Moondream3Processor._image_array_to_uint8(image_array) - if image_array.shape[-1] == 1: - image_array = image_array[..., 0] - - return Image.fromarray(image_array) - - def preprocess_image( - self, - image: ImageInput, - max_crops: int = 12, - overlap_margin: int = 4, - crop_size: int = 378, - patch_size: int = 14, - convert_to_rgb: bool = True, - return_tensors: str = "pt", - ) -> tuple[torch.Tensor, tuple[int, int]]: - """Preprocess an image using overlap-and-resize cropping strategy. - - Args: - image: Input PIL image, numpy array, or torch tensor. - max_crops: Maximum number of crops. - overlap_margin: Margin for overlapping in patches. - crop_size: Size of each crop. - patch_size: Size of each patch. - convert_to_rgb: Whether to convert to RGB. - return_tensors: Return type ("pt" for PyTorch). - - Returns: - Tuple of (pixel_values tensor, tiling tuple). - - """ - image = self._to_pil_image(image) - if convert_to_rgb: - image = convert_image_mode(image, "RGB") - - # Convert to numpy array - image_array = np.array(image) - original_h, original_w = image_array.shape[:2] - - margin_pixels = patch_size * overlap_margin - total_margin_pixels = margin_pixels * 2 - - crop_patches = crop_size // patch_size - crop_window_patches = crop_patches - (2 * overlap_margin) - crop_window_size = crop_window_patches * patch_size - - tiling = select_tiling( - original_h - total_margin_pixels, - original_w - total_margin_pixels, - crop_window_size, - max_crops, - ) - - n_crops = tiling[0] * tiling[1] + 1 - crops = np.zeros((n_crops, crop_size, crop_size, 3), dtype=np.uint8) - - target_size = ( - tiling[0] * crop_window_size + total_margin_pixels, - tiling[1] * crop_window_size + total_margin_pixels, - ) - - # Resize image - pil_img = Image.fromarray(image_array) - resized = pil_img.resize( - (int(target_size[1]), int(target_size[0])), - resample=Image.Resampling.LANCZOS, - ) - resized_array = np.asarray(resized) - - # Create global crop - global_pil = pil_img.resize( - (crop_size, crop_size), resample=Image.Resampling.LANCZOS - ) - crops[0] = np.asarray(global_pil) - - # Create local crops - for i in range(tiling[0]): - for j in range(tiling[1]): - y0 = i * crop_window_size - x0 = j * crop_window_size - y_end = min(y0 + crop_size, resized_array.shape[0]) - x_end = min(x0 + crop_size, resized_array.shape[1]) - - crop_region = resized_array[y0:y_end, x0:x_end] - crop_idx = 1 + i * tiling[1] + j - h_slice = slice(None, crop_region.shape[0]) - w_slice = slice(None, crop_region.shape[1]) - crops[crop_idx, h_slice, w_slice] = crop_region - - # Convert to tensor: (n_crops, H, W, C) -> (n_crops, C, H, W) - pixel_values = np.transpose(crops, (0, 3, 1, 2)) - - if return_tensors == "pt": - # Match HF reference preprocessing exactly: convert uint8 crops to - # bfloat16 before in-place normalization. - pixel_values = ( - torch.from_numpy(pixel_values) - .to(dtype=torch.bfloat16) - .div_(255.0) - .sub_(0.5) - .div_(0.5) - ) - else: - pixel_values = pixel_values.astype(np.float32) / 255.0 - pixel_values = (pixel_values - 0.5) / 0.5 - - return pixel_values, tiling - - def get_num_image_tokens(self) -> int: - """Return the number of image tokens (729 = 27x27 patches).""" - return self.patches_per_crop - - def batch_decode(self, *args, **kwargs): - """Forward to tokenizer's batch_decode.""" - return self.tokenizer.batch_decode(*args, **kwargs) - - def decode(self, *args, **kwargs): - """Forward to tokenizer's decode.""" - return self.tokenizer.decode(*args, **kwargs) - - @property - def model_input_names(self): - tokenizer_input_names = self.tokenizer.model_input_names - return tokenizer_input_names + ["pixel_values", "tilings"] - - -AutoProcessor.register("Moondream3Processor", Moondream3Processor) diff --git a/vllm/transformers_utils/processors/nemotron_vl.py b/vllm/transformers_utils/processors/nemotron_vl.py deleted file mode 100644 index fefb05cff4..0000000000 --- a/vllm/transformers_utils/processors/nemotron_vl.py +++ /dev/null @@ -1,372 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -import torch -import torchvision.transforms as T -from PIL import Image - -from vllm.multimodal.image import convert_image_mode -from vllm.tokenizers.hf import HfTokenizer - -from .internvl import InternVLImageProcessor, InternVLProcessor - - -def build_transform(input_size: int): - return T.Compose( - [ - T.Lambda(lambda img: convert_image_mode(img, "RGB")), - T.Resize( - (input_size, input_size), interpolation=T.InterpolationMode.BICUBIC - ), - T.ToTensor(), - ] - ) - - -# adapted from https://huggingface.co/nvidia/Llama-3.1-Nemotron-Nano-VL-8B-V1 -def find_closest_aspect_ratio( - aspect_ratio: float, - target_ratios: list[tuple[int, int]], - *, - width: int, - height: int, - image_size: int, -) -> tuple[int, int]: - best_factor = float("-inf") - best_ratio = (1, 1) - area = width * height - - for rw, rh in target_ratios: - target_aspect_ratio = rw / rh - size_factor = min((rw * rh * image_size * image_size) / area, 0.6) - ratio_closeness = min( - target_aspect_ratio / aspect_ratio, aspect_ratio / target_aspect_ratio - ) - factor = size_factor * ratio_closeness - - if factor > best_factor: - best_factor = factor - best_ratio = (rw, rh) - - return best_ratio - - -def calculate_nemotron_vl_targets( - *, - orig_width: int, - orig_height: int, - target_ratios: list[tuple[int, int]], - image_size: int, - use_thumbnail: bool, -) -> tuple[int, int, int]: - aspect_ratio = orig_width / orig_height - - # find the closest aspect ratio to the target - target_aspect_ratio = find_closest_aspect_ratio( - aspect_ratio, - target_ratios, - width=orig_width, - height=orig_height, - image_size=image_size, - ) - - # calculate the target width and height - target_width = image_size * target_aspect_ratio[0] - target_height = image_size * target_aspect_ratio[1] - blocks = target_aspect_ratio[0] * target_aspect_ratio[1] - - # add thumbnail image if num_blocks != 1 - if use_thumbnail and blocks != 1: - blocks += 1 - - return blocks, target_width, target_height - - -def dynamic_preprocess_nemotron_vl( - image: Image.Image, - *, - target_ratios: list[tuple[int, int]], - image_size: int, - use_thumbnail: bool, -) -> list[Image.Image]: - orig_width, orig_height = image.size - - # calculate the number of blocks without thumbnail - blocks, target_width, target_height = calculate_nemotron_vl_targets( - orig_width=orig_width, - orig_height=orig_height, - target_ratios=target_ratios, - image_size=image_size, - use_thumbnail=False, - ) - - # resize the image - resized_img = image.resize((target_width, target_height)) - processed_images = [] - for i in range(blocks): - box = ( - (i % (target_width // image_size)) * image_size, - (i // (target_width // image_size)) * image_size, - ((i % (target_width // image_size)) + 1) * image_size, - ((i // (target_width // image_size)) + 1) * image_size, - ) - # split the image - split_img = resized_img.crop(box) - processed_images.append(split_img) - - assert len(processed_images) == blocks - - if use_thumbnail and len(processed_images) != 1: - thumbnail_img = image.resize((image_size, image_size)) - processed_images.append(thumbnail_img) - - return processed_images - - -def get_nemotron_vl_target_ratios( - min_num: int, - max_num: int, -) -> list[tuple[int, int]]: - target_ratios = { - (i, j) - for n in range(min_num, max_num + 1) - for i in range(1, n + 1) - for j in range(1, n + 1) - if min_num <= i * j <= max_num - } - return sorted(target_ratios, key=lambda x: x[0] * x[1]) - - -def image_to_pixel_values_nemotron_vl( - image: Image.Image, - *, - input_size: int, - min_num: int, - max_num: int, - use_thumbnail: bool, - transform: T.Compose | None = None, -) -> torch.Tensor: - target_ratios = get_nemotron_vl_target_ratios(min_num, max_num) - - if transform is None: - transform = build_transform(input_size=input_size) - - images = dynamic_preprocess_nemotron_vl( - image, - target_ratios=target_ratios, - image_size=input_size, - use_thumbnail=use_thumbnail, - ) - - pixel_values = torch.stack([transform(image) for image in images]) - return pixel_values - - -class LlamaNemotronNanoVLImageProcessor(InternVLImageProcessor): - def _images_to_pixel_values_lst( - self, - images: list[Image.Image], - min_dynamic_patch: int | None = None, - max_dynamic_patch: int | None = None, - dynamic_image_size: bool | None = None, - ) -> list[torch.Tensor]: - min_num, max_num = self.resolve_min_max_num( - min_dynamic_patch=min_dynamic_patch, - max_dynamic_patch=max_dynamic_patch, - dynamic_image_size=dynamic_image_size, - use_thumbnail=False, # Applied in image_to_pixel_values - ) - - return [ - image_to_pixel_values_nemotron_vl( - image, - input_size=self.image_size, - min_num=min_num, - max_num=max_num, - use_thumbnail=self.use_thumbnail, - transform=build_transform(self.image_size), - ) - for image in images - ] - - -class LlamaNemotronNanoVLProcessor(InternVLProcessor): - """This model doesn't define its own HF processor, - so we implement our own one here. - - The image processor is given by: - https://huggingface.co/nvidia/Llama-3.1-Nemotron-Nano-VL-8B-V1/blob/main/image_processing.py - """ - - def __init__( - self, - image_processor: LlamaNemotronNanoVLImageProcessor, - tokenizer: HfTokenizer, - *, - image_seq_length: int, - start_image_token: str = "", - end_image_token: str = "", - ctx_image_token: str = "", - ) -> None: - super().__init__( - image_processor=image_processor, - tokenizer=tokenizer, - image_seq_length=image_seq_length, - start_image_token=start_image_token, - end_image_token=end_image_token, - ctx_image_token=ctx_image_token, - ) - - def get_num_image_tokens( - self, - *, - image_width: int, - image_height: int, - ) -> int: - image_processor = self.image_processor - target_ratios = self.resolve_target_ratios( - use_thumbnail=False, # Applied in calculate_targets - ) - - num_patches, _, _ = calculate_nemotron_vl_targets( - orig_width=image_width, - orig_height=image_height, - image_size=image_processor.image_size, - target_ratios=target_ratios, - use_thumbnail=image_processor.use_thumbnail, - ) - - return num_patches * self.image_seq_length - - -# SigLIP normalization constants -SIGLIP_MEAN = (0.5, 0.5, 0.5) -SIGLIP_STD = (0.5, 0.5, 0.5) - - -def build_siglip_transform( - input_size: int, - *, - normalize_on_cpu: bool = True, -): - """Build transform for SigLIP vision encoder with normalization. - - Extends the base transform from nemotron_vl with SigLIP-specific normalization. - """ - transforms = [ - T.Lambda(lambda img: convert_image_mode(img, "RGB")), - T.Resize((input_size, input_size), interpolation=T.InterpolationMode.BICUBIC), - T.ToTensor() if normalize_on_cpu else T.PILToTensor(), - ] - if normalize_on_cpu: - transforms.append(T.Normalize(mean=SIGLIP_MEAN, std=SIGLIP_STD)) - - return T.Compose(transforms) - - -class LlamaNemotronVLEmbedImageProcessor(InternVLImageProcessor): - def __init__( - self, - image_size: int, - min_dynamic_patch: int, - max_dynamic_patch: int, - dynamic_image_size: bool, - use_thumbnail: bool, - *, - do_rescale: bool = True, - do_normalize: bool = True, - ) -> None: - if do_rescale != do_normalize: - raise ValueError( - "LlamaNemotronVLEmbedImageProcessor requires do_rescale and " - "do_normalize to be both enabled or both disabled." - ) - - super().__init__( - image_size=image_size, - min_dynamic_patch=min_dynamic_patch, - max_dynamic_patch=max_dynamic_patch, - dynamic_image_size=dynamic_image_size, - use_thumbnail=use_thumbnail, - ) - self.normalize_on_cpu = do_normalize - - def _images_to_pixel_values_lst( - self, - images: list[Image.Image], - min_dynamic_patch: int | None = None, - max_dynamic_patch: int | None = None, - dynamic_image_size: bool | None = None, - ) -> list[torch.Tensor]: - min_num, max_num = self.resolve_min_max_num( - min_dynamic_patch=min_dynamic_patch, - max_dynamic_patch=max_dynamic_patch, - dynamic_image_size=dynamic_image_size, - use_thumbnail=False, # Applied in image_to_pixel_values - ) - - return [ - image_to_pixel_values_nemotron_vl( - image, - input_size=self.image_size, - min_num=min_num, - max_num=max_num, - use_thumbnail=self.use_thumbnail, - transform=build_siglip_transform( - self.image_size, - normalize_on_cpu=self.normalize_on_cpu, - ), - ) - for image in images - ] - - -class LlamaNemotronVLEmbedProcessor(InternVLProcessor): - """Processor for LlamaNemotronVL embedding model. - - Inherits from NemotronVLProcessor and specializes it for embedding tasks: - - Uses SigLIP transform with normalization instead of base transform - - Uses different image context token ( vs ) - """ - - def __init__( - self, - image_processor: LlamaNemotronVLEmbedImageProcessor, - tokenizer: HfTokenizer, - *, - image_seq_length: int, - start_image_token: str = "", - end_image_token: str = "", - ctx_image_token: str = "", - ) -> None: - super().__init__( - image_processor=image_processor, - tokenizer=tokenizer, - image_seq_length=image_seq_length, - start_image_token=start_image_token, - end_image_token=end_image_token, - ctx_image_token=ctx_image_token, - ) - - self.image_processor: LlamaNemotronVLEmbedImageProcessor - - def get_num_image_tokens( - self, - *, - image_width: int, - image_height: int, - ) -> int: - image_processor = self.image_processor - target_ratios = self.resolve_target_ratios( - use_thumbnail=False, # Applied in calculate_targets - ) - - num_patches, _, _ = calculate_nemotron_vl_targets( - orig_width=image_width, - orig_height=image_height, - image_size=image_processor.image_size, - target_ratios=target_ratios, - use_thumbnail=image_processor.use_thumbnail, - ) - - return num_patches * self.image_seq_length diff --git a/vllm/transformers_utils/processors/nvlm_d.py b/vllm/transformers_utils/processors/nvlm_d.py deleted file mode 100644 index 91e446aadc..0000000000 --- a/vllm/transformers_utils/processors/nvlm_d.py +++ /dev/null @@ -1,63 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -# adapted from https://huggingface.co/nvidia/NVLM-D-72B/blob/main/modeling_nvlm_d.py -# -------------------------------------------------------- -# NVLM-D -# Copyright (c) 2024 NVIDIA -# Licensed under Apache 2.0 License [see LICENSE for details] -# -------------------------------------------------------- -from vllm.multimodal.processing import PromptUpdateDetails, cached_encode -from vllm.tokenizers.hf import HfTokenizer - -from .internvl import InternVLImageProcessor, InternVLProcessor - - -class NVLMProcessor(InternVLProcessor): - def __init__( - self, - image_processor: InternVLImageProcessor, - tokenizer: HfTokenizer, - *, - image_seq_length: int, - start_image_token: str = "", - end_image_token: str = "", - ctx_image_token: str = "<|vision_pad|>", - ) -> None: - super().__init__( - image_processor=image_processor, - tokenizer=tokenizer, - image_seq_length=image_seq_length, - start_image_token=start_image_token, - end_image_token=end_image_token, - ctx_image_token=ctx_image_token, - ) - - def get_image_repl( - self, - num_patches: int | None, - num_features: int | None = None, - ) -> PromptUpdateDetails: - if num_patches is None: - raise NotImplementedError("Embedding inputs are not supported") - - num_features = num_patches * self.image_seq_length - - tile_pos_identifiers = [f"" for i in range(1, num_patches)] - if self.image_processor.use_thumbnail: - tile_pos_identifiers += [""] - - context_size = num_features // num_patches - features = "".join( - (identifier + self.ctx_image_token * context_size) - for identifier in tile_pos_identifiers - ) - - # We include the start and end as well because "<", "tile"], resulting in assertion error - # when trying to find " Image.Image: - if isinstance(image, Image.Image): - return image.convert("RGB") - - if isinstance(image, torch.Tensor): - image = image.detach().cpu().numpy() - if not isinstance(image, np.ndarray): - raise TypeError( - "OpenVLA image input must be a PIL image, numpy array, or torch tensor; " - f"got {type(image)}" - ) - - if image.ndim != 3: - raise ValueError( - f"OpenVLA image input must have 3 dimensions, got shape {image.shape}" - ) - - if image.shape[0] in (1, 3): - image = np.moveaxis(image, 0, -1) - - if image.shape[-1] == 1: - image = np.repeat(image, 3, axis=-1) - elif image.shape[-1] != 3: - raise ValueError( - f"OpenVLA image input must have 1 or 3 channels, got shape {image.shape}" - ) - - if image.dtype != np.uint8: - image = image.astype(np.float32) - if image.max(initial=0.0) <= 1.0: - image = image * 255.0 - image = np.clip(image, 0, 255).astype(np.uint8) - - return Image.fromarray(image).convert("RGB") - - -def preprocess_openvla_image(image: Any, image_size: int) -> torch.Tensor: - rgb_image = to_rgb_image(image) - rgb_image = rgb_image.resize( - (image_size, image_size), - Image.Resampling.BICUBIC, - ) - - raw = np.asarray(rgb_image, dtype=np.float32) / 255.0 - dinov2_pixels = ((raw - IMAGENET_MEAN) / IMAGENET_STD).transpose(2, 0, 1) - siglip_pixels = ((raw - SIGLIP_MEAN) / SIGLIP_STD).transpose(2, 0, 1) - pixel_values = np.concatenate([dinov2_pixels, siglip_pixels], axis=0) - return torch.from_numpy(pixel_values) - - -class OpenVLAImageProcessor(ImageProcessingMixin): - def __init__(self, *, image_size: int) -> None: - self.image_size = image_size - - def __call__( - self, - images: Any | None = None, - **kwargs: object, - ) -> dict[str, object]: - if images is None: - return {} - if not isinstance(images, Sequence) or isinstance(images, (str, bytes)): - images = [images] - if len(images) == 0: - return {} - - pixel_values = torch.stack( - [ - preprocess_openvla_image(image, image_size=self.image_size) - for image in images - ], - dim=0, - ) - return {"pixel_values": pixel_values} - - -class OpenVLAProcessor(ProcessorMixin): - def __init__( - self, - *, - image_processor: OpenVLAImageProcessor, - tokenizer: Any, - ) -> None: - self.image_processor = image_processor - self.tokenizer = tokenizer diff --git a/vllm/transformers_utils/processors/ovis.py b/vllm/transformers_utils/processors/ovis.py deleted file mode 100644 index 9ad320be39..0000000000 --- a/vllm/transformers_utils/processors/ovis.py +++ /dev/null @@ -1,457 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -# ruff: noqa: E501 -# coding=utf-8 -# adapted from https://github.com/AIDC-AI/Ovis/blob/35ab51a1a1e3542fa6db260a1084cefbc8f164bb/ovis/vllm/processing_ovis.py -# Copyright 2025 The Qwen Team and The HuggingFace Inc. team. All rights reserved. -# -# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX -# and OPT implementations in this library. It has been modified from its -# original forms to accommodate minor architectural differences compared -# to GPT-NeoX and OPT used by the Meta AI team that trained the model. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from functools import cached_property - -import PIL -import torch -from transformers import BatchFeature -from transformers.image_utils import ImageInput -from transformers.processing_utils import ProcessingKwargs, ProcessorMixin, Unpack -from transformers.tokenization_utils_base import PreTokenizedInput, TextInput - -from vllm.multimodal.image import convert_image_mode - -__all__ = ["OvisProcessor"] -IGNORE_ID = -100 - - -class OvisProcessorKwargs(ProcessingKwargs, total=False): # type: ignore[call-arg] - _defaults = { - "text_kwargs": { - "padding": False, - }, - "images_kwargs": { - "do_convert_rgb": True, - "return_tensors": "pt", - }, - } - - -class OvisProcessor(ProcessorMixin): - r"""Constructs an Ovis processor which wraps an Ovis image processor and a Qwen2 tokenizer into a single processor. - [`OvisProcessor`] offers all the functionalities of [`Qwen2VLImageProcessor`] and [`Qwen2TokenizerFast`]. See the - [`~OvisProcessor.__call__`] and [`~OvisProcessor.decode`] for more information. - - Args: - image_processor ([`Qwen2VLImageProcessor`], *optional*): - The image processor is a required input. - tokenizer ([`Qwen2TokenizerFast`], *optional*): - The tokenizer is a required input. - chat_template (`str`, *optional*): A Jinja template which will be used to convert lists of messages - in a chat into a tokenizable string. - - """ - - attributes = ["image_processor", "tokenizer"] - valid_kwargs = ["chat_template", "image_pad_token", "image_segment_len"] - - image_processor_class = "AutoImageProcessor" - tokenizer_class = "AutoTokenizer" - - def __init__( - self, - image_processor=None, - tokenizer=None, - chat_template=None, - image_pad_token=None, - image_segment_len=255, - **kwargs, - ): - self.image_token = "" - self.image_pad_token = image_pad_token - self.image_segment_len = image_segment_len - super().__init__(image_processor, tokenizer, chat_template=chat_template) - - @cached_property - def extra_special_tokens(self): - image_pad_token_id = self.tokenizer.get_vocab()[self.image_pad_token] - extra_special_tokens = { - "image_token": -200, - "image_atom": -300, - "image_start": -301, - "image_prefix": -302, - "image_col_sep": -303, - "image_row_sep": -304, - "image_end": -305, - "image_pad": image_pad_token_id, - } - return extra_special_tokens - - def __call__( - self, - images: ImageInput = None, - text: TextInput - | PreTokenizedInput - | list[TextInput] - | list[PreTokenizedInput] = None, - **kwargs: Unpack[OvisProcessorKwargs], - ) -> BatchFeature: - """Main method to prepare for the model one or several sequences(s) and image(s). This method forwards the `text` - and `kwargs` arguments to Qwen2TokenizerFast's [`~Qwen2TokenizerFast.__call__`] if `text` is not `None` to encode - the text. To prepare the vision inputs, this method forwards the `vision_infos` and `kwargs` arguments to - Qwen2VLImageProcessor's [`~Qwen2VLImageProcessor.__call__`] if `vision_infos` is not `None`. - - Args: - images (`PIL.Image.Image`, `np.ndarray`, `torch.Tensor`, `list[PIL.Image.Image]`, `list[np.ndarray]`, `list[torch.Tensor]`): - The image or batch of images to be prepared. Each image can be a PIL image, NumPy array or PyTorch - tensor. Both channels-first and channels-last formats are supported. - text (`str`, `list[str]`, `list[list[str]]`): - The sequence or batch of sequences to be encoded. Each sequence can be a string or a list of strings - (pretokenized string). If the sequences are provided as list of strings (pretokenized), you must set - `is_split_into_words=True` (to lift the ambiguity with a batch of sequences). - videos (`np.ndarray`, `torch.Tensor`, `list[np.ndarray]`, `list[torch.Tensor]`): - The image or batch of videos to be prepared. Each video can be a 4D NumPy array or PyTorch - tensor, or a nested list of 3D frames. Both channels-first and channels-last formats are supported. - return_tensors (`str` or [`~utils.TensorType`], *optional*): - If set, will return tensors of a particular framework. Acceptable values are: - - `'tf'`: Return TensorFlow `tf.constant` objects. - - `'pt'`: Return PyTorch `torch.Tensor` objects. - - `'np'`: Return NumPy `np.ndarray` objects. - - `'jax'`: Return JAX `jnp.ndarray` objects. - - Returns: - [`BatchFeature`]: A [`BatchFeature`] with the following fields: - - **input_ids** -- List of token ids to be fed to a model. Returned when `text` is not `None`. - - **attention_mask** -- List of indices specifying which tokens should be attended to by the model (when - `return_attention_mask=True` or if *"attention_mask"* is in `self.model_input_names` and if `text` is not - `None`). - - **pixel_values** -- Pixel values to be fed to a model. Returned when `images` is not `None`. - - **pixel_values_videos** -- Pixel values of videos to be fed to a model. Returned when `videos` is not `None`. - - **image_grid_thw** -- List of image 3D grid in LLM. Returned when `images` is not `None`. - - **video_grid_thw** -- List of video 3D grid in LLM. Returned when `videos` is not `None`. - - **second_per_grid_ts** -- List of video seconds per time grid. Returned when `videos` is not `None`. - - """ - max_partition = kwargs.pop("max_partition", 9) - covering_threshold = kwargs.pop("covering_threshold", 0.9) - - output_kwargs = self._merge_kwargs( - OvisProcessorKwargs, - tokenizer_init_kwargs=self.tokenizer.init_kwargs, - **kwargs, - ) - - # Process all images first - image_features = {} - if images is not None: - processed_images = [] - image_placeholders_list = [] - grids = [] - - # Process each image - for image in images if isinstance(images, list) else [images]: - pixel_values, image_placeholders, grid = self.preprocess_image( - image=image, - max_partition=max_partition, - covering_threshold=covering_threshold, - **output_kwargs["images_kwargs"], - ) - processed_images.append(pixel_values) - image_placeholders_list.append(image_placeholders) - grids.append(grid) - - # assign all processed images - if processed_images: - image_features["image_placeholders"] = image_placeholders_list - - # Process text input - if text is not None: - if not isinstance(text, list): - text = [text] - - tokenized_batched_text = self._tokenize_with_image_symbol(text) - image_token_id = self.get_token_value("image_token") - replaced_ids_list = [] - idx = 0 - for ids_tensor in tokenized_batched_text: - if ( - image_token_id in ids_tensor - and "image_placeholders" in image_features - ): - if idx < len(image_features["image_placeholders"]): - # Converts in list for ease of use - ids_list = ids_tensor.tolist() - - new_ids = [] - - # replace placeholders - for i, token_id in enumerate(ids_list): - if token_id == image_token_id: - placeholder_ids = image_features["image_placeholders"][ - idx - ] - new_ids.extend(placeholder_ids) - idx += 1 - else: - new_ids.append(token_id) - - # Converts back to tensors - ids_tensor = torch.tensor(new_ids, dtype=torch.long) - else: - raise RuntimeError( - "Mismatch between the images you provided and the number of placeholder present in the text" - ) - - replaced_ids_list.append(ids_tensor) - - if replaced_ids_list: - replaced_and_tokenized_ids = torch.stack(replaced_ids_list) - else: - replaced_and_tokenized_ids = torch.tensor([], dtype=torch.long) - - # Create the output with text features - output = BatchFeature( - data={ - "input_ids": replaced_and_tokenized_ids, - } - ) - - # Add image features if present - if image_features: - output["pixel_values"] = processed_images - output["grids"] = grids - - return output - - # If only images were provided - return BatchFeature(data=image_features) - - def _tokenize_with_image_symbol(self, text_list: list[str]) -> torch.LongTensor: - batch_token_ids = [] - for text in text_list: - text_chunks = [ - self.tokenizer(chunk, add_special_tokens=False).input_ids - for chunk in text.split(self.image_token) - ] - token_ids = [] - num_chuck = len(text_chunks) - for i, chunk in enumerate(text_chunks): - token_ids.extend(chunk) - if i < num_chuck - 1: - token_ids.append(self.get_token_value("image_token")) - batch_token_ids.append(token_ids) - return torch.tensor(batch_token_ids, dtype=torch.long) - - def get_image_size(self): - size = self.image_processor.size - if "shortest_edge" in size: - width = height = size["shortest_edge"] - elif "height" in size and "width" in size: - width = size["width"] - height = size["height"] - else: - raise ValueError("Can't parse image size from image_processor config.") - return height, width - - def get_token_value(self, tok): - return self.extra_special_tokens[tok] - - def construct_image_indicators(self, grid): - image_placeholders = [ - self.get_token_value("image_start"), - self.get_token_value("image_atom"), - self.get_token_value("image_prefix"), - ] - if grid[0] * grid[1] > 1: - for r in range(grid[0]): - for c in range(grid[1]): - image_placeholders.append(self.get_token_value("image_atom")) - if c < grid[1] - 1: - image_placeholders.append(self.get_token_value("image_col_sep")) - if r < grid[0] - 1: - image_placeholders.append(self.get_token_value("image_row_sep")) - image_placeholders.append(self.get_token_value("image_end")) - return image_placeholders - - def construct_image_placeholders(self, grid): - image_placeholders = self.construct_image_indicators(grid) - - image_atom_token_id = self.get_token_value("image_atom") - # Extract the padding token ID from tokenizer - image_padding_token_id = self.get_token_value("image_pad") - - # Create a new list with padding tokens inserted - padded_placeholder_tokens = [] - for token in image_placeholders: - padded_placeholder_tokens.append(image_padding_token_id) - if token == image_atom_token_id: - padded_placeholder_tokens.extend( - [image_padding_token_id] * self.image_segment_len - ) - return padded_placeholder_tokens - - def preprocess_image( - self, - image: PIL.Image.Image, - max_partition, - covering_threshold, - do_convert_rgb, - return_tensors, - ): - def _preprocess(img: PIL.Image.Image, side): - # first resize and preprocess - w, h = img.size - if w == h: - new_width = new_height = side - elif w > h: - new_width = side - new_height = int(h / w * new_width) - else: - new_height = side - new_width = int(w / h * new_height) - new_size = dict(height=new_height, width=new_width) - pixel_values = self.image_processor.preprocess( - img, size=new_size, return_tensors=return_tensors - )["pixel_values"] - - # then pad to square - square_values = torch.zeros( - [1, 3, side, side], dtype=pixel_values.dtype, device=pixel_values.device - ) - new_height, new_width = pixel_values.shape[2:] - if new_height == new_width: - square_values[:, :, :, :] = pixel_values - elif new_height > new_width: - from_index = (side - new_width) // 2 - square_values[:, :, :, from_index : from_index + new_width] = ( - pixel_values - ) - else: - from_index = (side - new_height) // 2 - square_values[:, :, from_index : from_index + new_height, :] = ( - pixel_values - ) - - return square_values - - def _partition(img, grid) -> list[tuple[int, int, int, int]]: - w, h = img.size - row_height = h // grid[0] - col_width = w // grid[1] - - partition = [] - for row in range(grid[0]): - for col in range(grid[1]): - left = col * col_width - upper = row * row_height - right = w if col == grid[1] - 1 else (col + 1) * col_width - lower = h if row == grid[0] - 1 else (row + 1) * row_height - partition.append((left, upper, right, lower)) - - return partition - - def _covering_area(left, upper, right, lower, side): - w = right - left - h = lower - upper - w, h = max(w, h), min(w, h) - if w > side: - h = h / w * side - w = side - return w * h - - def _get_best_grid(img, side): - img_area = img.size[0] * img.size[1] - - candidate_grids = [] - for i in range(1, max_partition + 1): - for j in range(1, max_partition + 1): - if i * j <= max_partition: - candidate_grids.append((i, j)) - - all_grids = [] - good_grids = [] - for grid in candidate_grids: - partition = _partition(img, grid) - covering_ratio = ( - sum([_covering_area(*p, side) for p in partition]) / img_area - ) - assert covering_ratio <= 1.0 - all_grids.append((grid, covering_ratio)) - if covering_ratio > covering_threshold: - good_grids.append((grid, covering_ratio)) - - if len(good_grids) > 0: - # pick the good partition with minimum #sub_images and break the tie using covering_ratio - return sorted(good_grids, key=lambda x: (x[0][0] * x[0][1], -x[1]))[0][ - 0 - ] - else: - # pick the partition with maximum covering_ratio and break the tie using #sub_images - return sorted(all_grids, key=lambda x: (-x[1], x[0][0] * x[0][1]))[0][0] - - if do_convert_rgb: - image = convert_image_mode(image, "RGB") - - sides = self.get_image_size() - if sides[0] != sides[1]: - raise ValueError("get_image_size() returns non-square size") - side = sides[0] - grid = _get_best_grid(image, side) - partition = _partition(image, grid) - crops = [image.crop(p) for p in partition] - if len(crops) > 1: - crops.insert(0, image) - pixel_values = torch.cat([_preprocess(crop, side) for crop in crops], dim=0) - image_placeholders = self.construct_image_placeholders(grid) - return torch.tensor(pixel_values), image_placeholders, torch.tensor(grid) - - def batch_decode(self, *args, **kwargs): - """This method forwards all its arguments to Qwen2TokenizerFast's [`~PythonBackend.batch_decode`]. Please - refer to the docstring of this method for more information. - """ - return self.tokenizer.batch_decode(*args, **kwargs) - - def decode(self, *args, **kwargs): - """This method forwards all its arguments to Qwen2TokenizerFast's [`~PythonBackend.decode`]. Please refer to - the docstring of this method for more information. - """ - return self.tokenizer.decode(*args, **kwargs) - - def post_process_image_text_to_text(self, generated_outputs): - """Post-process the output of the model to decode the text. - - Args: - generated_outputs (`torch.Tensor` or `np.ndarray`): - The output of the model `generate` function. The output is expected to be a tensor of shape `(batch_size, sequence_length)` - or `(sequence_length,)`. - - Returns: - `list[str]`: The decoded text. - - """ - return self.tokenizer.batch_decode( - generated_outputs, - skip_special_tokens=True, - clean_up_tokenization_spaces=False, - ) - - @property - def model_input_names(self): - tokenizer_input_names = self.tokenizer.model_input_names - image_processor_input_names = self.image_processor.model_input_names - names_from_processor = list( - dict.fromkeys(tokenizer_input_names + image_processor_input_names) - ) - return names_from_processor + ["second_per_grid_ts"] diff --git a/vllm/transformers_utils/processors/ovis2_5.py b/vllm/transformers_utils/processors/ovis2_5.py deleted file mode 100644 index 9e4e47738a..0000000000 --- a/vllm/transformers_utils/processors/ovis2_5.py +++ /dev/null @@ -1,476 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -import math -from functools import cached_property - -import numpy as np -import PIL -import torch -from transformers import BatchFeature -from transformers.image_utils import ImageInput -from transformers.processing_utils import ProcessingKwargs, ProcessorMixin, Unpack -from transformers.tokenization_utils_base import PreTokenizedInput, TextInput - -__all__ = ["Ovis2_5Processor"] -IMAGE_TOKEN = "" -VIDEO_TOKEN = "