From 2066d3143f8e3231be60863eb0e2fc0f1aad1fdf Mon Sep 17 00:00:00 2001 From: MOVZX Date: Mon, 19 Oct 2026 17:10:56 +0700 Subject: [PATCH] [Mini-VLLM] Cut non-OpenAI surfaces, dead kernels, Mistral stack - Remove Anthropic, Cohere, SageMaker, decisions, structured-decisions, rl, render-launcher and batch entrypoints; keep scale-out token routes - Drop the Swagger static bundle and the /docs, /redoc routes - Delete device-keyed tuning configs for fused_moe, quant utils, mamba select-state and helion; no sm86 entry ever existed for any of them - Remove ten unused rope variants plus the fbgemm_fp8 and fp_quant quantizers and their config, metrics and loader plumbing - Cut the CPU device build: csrc/cpu, cpu_extension.cmake, cpu requirements and the setup.py cpu target path - Delete 29 environment variables nothing reads and 14 chat-template fallbacks for models this fork no longer registers - Remove nine dead CUDA ops (deepseek_v4, kimi_k3, minimax, muse_spark, dsv3 gemm) with their kernels, bindings, wrappers and CMake entries - Delete the connector test suites that targeted removed connectors - Cut the Mistral tokenizer and parser stack, its tests, and the mistral_common and ijson dependency pins - Restore the Qwen4Exp registry rows and nvidia/mtp.py that an earlier cut deleted while keeping the model tree Co-authored-by: Hinata Signed-off-by: MOVZX --- .agents/skills/transformers-upgrade/SKILL.md | 5 +- CMakeLists.txt | 77 +- cmake/cpu_extension.cmake | 698 - csrc/cpu/activation.cpp | 175 - csrc/cpu/activation_lut_bf16.cpp | 71 - csrc/cpu/cpu_arch_macros.h | 197 - csrc/cpu/cpu_attn.cpp | 287 - csrc/cpu/cpu_attn_amx.hpp | 636 - csrc/cpu/cpu_attn_amx_fp8.hpp | 795 - csrc/cpu/cpu_attn_fp8.hpp | 187 - csrc/cpu/cpu_attn_impl.hpp | 2190 --- csrc/cpu/cpu_attn_neon.hpp | 402 - csrc/cpu/cpu_attn_neon_bfmmla.hpp | 556 - csrc/cpu/cpu_attn_rvv.hpp | 412 - csrc/cpu/cpu_attn_vec.hpp | 327 - csrc/cpu/cpu_attn_vec16.hpp | 171 - csrc/cpu/cpu_attn_vsx.hpp | 366 - csrc/cpu/cpu_attn_vxe.hpp | 387 - csrc/cpu/cpu_fused_moe.cpp | 721 - csrc/cpu/cpu_fused_moe_activations.hpp | 215 - csrc/cpu/cpu_fused_moe_int8.cpp | 660 - csrc/cpu/cpu_isa.cpp | 20 - csrc/cpu/cpu_tanhf_neon.hpp | 128 - csrc/cpu/cpu_types.hpp | 44 - csrc/cpu/cpu_types_arm.hpp | 992 - csrc/cpu/cpu_types_riscv.hpp | 25 - csrc/cpu/cpu_types_riscv_defs.hpp | 121 - csrc/cpu/cpu_types_riscv_impl.hpp | 1034 - csrc/cpu/cpu_types_scalar.hpp | 526 - csrc/cpu/cpu_types_vsx.hpp | 1186 -- csrc/cpu/cpu_types_vxe.hpp | 1450 -- csrc/cpu/cpu_types_x86.hpp | 1170 -- csrc/cpu/cpu_wna16.cpp | 412 - csrc/cpu/dnnl_helper.cpp | 575 - csrc/cpu/dnnl_helper.h | 223 - csrc/cpu/dnnl_kernels.cpp | 594 - csrc/cpu/dynamic_4bit_int_moe_cpu.cpp | 172 - csrc/cpu/float_convert.hpp | 104 - csrc/cpu/generate_cpu_attn_dispatch.py | 295 - csrc/cpu/layernorm.cpp | 136 - csrc/cpu/mamba_cpu.cpp | 285 - csrc/cpu/mamba_kernels.hpp | 382 - csrc/cpu/micro_gemm/cpu_micro_gemm_amx.hpp | 280 - csrc/cpu/micro_gemm/cpu_micro_gemm_impl.hpp | 135 - .../micro_gemm/cpu_micro_gemm_int8_neon.hpp | 424 - .../micro_gemm/cpu_micro_gemm_int8_vsx.hpp | 287 - csrc/cpu/micro_gemm/cpu_micro_gemm_neon.hpp | 509 - csrc/cpu/micro_gemm/cpu_micro_gemm_rvv.hpp | 228 - csrc/cpu/micro_gemm/cpu_micro_gemm_vec.hpp | 135 - csrc/cpu/micro_gemm/cpu_micro_gemm_vsx.hpp | 445 - csrc/cpu/mla_decode.cpp | 392 - csrc/cpu/ops.h | 46 - csrc/cpu/pos_encoding.cpp | 366 - csrc/cpu/sampling_kernels.cpp | 168 - csrc/cpu/sgl-kernels/blas_gemm.h | 82 - csrc/cpu/sgl-kernels/bmm.cpp | 208 - csrc/cpu/sgl-kernels/common.h | 549 - csrc/cpu/sgl-kernels/compressor.cpp | 747 - csrc/cpu/sgl-kernels/conv.cpp | 894 - csrc/cpu/sgl-kernels/conv_arm.h | 126 - csrc/cpu/sgl-kernels/decode.cpp | 1821 -- csrc/cpu/sgl-kernels/extend.cpp | 568 - csrc/cpu/sgl-kernels/fla.cpp | 2674 --- csrc/cpu/sgl-kernels/flash_attn.h | 250 - csrc/cpu/sgl-kernels/flash_mla.cpp | 625 - csrc/cpu/sgl-kernels/gemm.cpp | 844 - csrc/cpu/sgl-kernels/gemm.h | 449 - csrc/cpu/sgl-kernels/gemm_fp8.cpp | 1238 -- csrc/cpu/sgl-kernels/gemm_fp8_w8a8.cpp | 1099 -- csrc/cpu/sgl-kernels/gemm_int4.cpp | 1054 -- csrc/cpu/sgl-kernels/gemm_int8.cpp | 546 - csrc/cpu/sgl-kernels/indexer.cpp | 276 - csrc/cpu/sgl-kernels/mhc.cpp | 1046 - csrc/cpu/sgl-kernels/mla_cache.cpp | 111 - csrc/cpu/sgl-kernels/moe.cpp | 1461 -- csrc/cpu/sgl-kernels/moe.h | 243 - csrc/cpu/sgl-kernels/moe_fp8.cpp | 413 - csrc/cpu/sgl-kernels/moe_fp8_w8a8.cpp | 432 - csrc/cpu/sgl-kernels/moe_int4.cpp | 323 - csrc/cpu/sgl-kernels/moe_int8.cpp | 965 - csrc/cpu/sgl-kernels/paged_mqa_logits.cpp | 217 - csrc/cpu/sgl-kernels/store_cache.cpp | 481 - csrc/cpu/sgl-kernels/topk.cpp | 834 - csrc/cpu/sgl-kernels/vec.h | 643 - csrc/cpu/sgl-kernels/vec_pack.h | 299 - csrc/cpu/shm.cpp | 869 - csrc/cpu/spec_decode_utils.cpp | 492 - csrc/cpu/torch_bindings.cpp | 1072 -- csrc/cpu/utils.cpp | 149 - csrc/cpu/utils.hpp | 146 - csrc/dsv3_fused_a_gemm.cu | 865 - ...deepseek_v4_qnorm_rope_kv_insert_kernel.cu | 1390 -- ..._kimi_k3_mla_key_concat_kv_cache_kernel.cu | 1603 -- ...minimax_m3_qknorm_rope_kv_insert_kernel.cu | 1218 -- ...muse_spark_qknorm_rope_kv_insert_kernel.cu | 752 - csrc/fused_qknorm_rope_cache_kernel.cu | 567 - csrc/kimi_k3/attn_res_kernel.cu | 1033 - csrc/minimax_reduce_rms_kernel.cu | 863 - csrc/minimax_reduce_rms_kernel.h | 79 - csrc/muse_spark_mul_tanh_soft_cap_kernel.cu | 100 - csrc/ops.h | 177 - csrc/torch_bindings.cpp | 191 - pyproject.toml | 8 +- requirements/common.txt | 3 - requirements/cpu.txt | 29 - requirements/test/cpu.txt | 1366 -- requirements/test/cuda.in | 2 - requirements/test/rocm.in | 2 - requirements/test/xpu.in | 2 - setup.py | 101 +- tests/entrypoints/anthropic/__init__.py | 0 .../test_anthropic_messages_conversion.py | 2199 --- tests/entrypoints/anthropic/test_messages.py | 245 - .../anthropic/test_protocol_exports.py | 52 - tests/entrypoints/cohere/__init__.py | 0 tests/entrypoints/cohere/test_api_router.py | 693 - tests/entrypoints/cohere/test_chat_v2.py | 343 - .../cohere/test_cohere_chat_message.py | 135 - tests/entrypoints/cohere/test_protocol.py | 271 - .../cohere/test_registry_and_args.py | 91 - .../cohere/test_serving_conversion.py | 2569 --- .../cohere/test_serving_streaming.py | 952 - .../generate/structured_decisions/__init__.py | 0 .../structured_decisions/test_decisions.py | 240 - .../test_question_types.py | 179 - .../test_structured_decisions_e2e.py | 260 - .../test_api_server_process_manager.py | 196 - .../entrypoints/launchers/render/__init__.py | 2 - .../launchers/render/test_app_state.py | 84 - tests/entrypoints/launchers/test_cli_args.py | 135 +- tests/entrypoints/launchers/test_run_batch.py | 956 - .../llm/offline_mode/test_offline_mode.py | 10 - .../openai/chat_completion/test_chat_error.py | 53 - .../openai/decisions/test_decisions_e2e.py | 101 - .../openai/test_render_token_offsets.py | 4 +- tests/entrypoints/rl/__init__.py | 0 tests/entrypoints/rl/conftest.py | 349 - .../rl/return_routed_experts/__init__.py | 0 .../return_routed_experts/test_openai_api.py | 73 - .../test_route_request_bodies.py | 101 - .../test_scale_out_api.py | 88 - .../rl/state_transitions/__init__.py | 0 .../rl/state_transitions/test_pause_resume.py | 168 - .../rl/state_transitions/test_sleep.py | 306 - .../rl/weight_transfer/__init__.py | 3 - .../rl/weight_transfer/test_offline.py | 341 - .../rl/weight_transfer_metrics/__init__.py | 0 .../test_weight_operation_metrics.py | 189 - .../scale_out/derender/__init__.py | 0 .../derender/test_derender_parity.py | 880 - tests/entrypoints/scale_out/derender/utils.py | 118 - .../scale_out/ec_integration/README.md | 43 - .../run_scale_out_ec_e2e_test.sh | 277 - .../ec_integration/test_scale_out_ec_e2e.py | 276 - .../entrypoints/scale_out/render/__init__.py | 0 .../scale_out/render/test_launch_render.py | 179 - .../scale_out/render/test_render.py | 882 - .../render/test_render_multimodal.py | 147 - tests/entrypoints/scale_out/test_factories.py | 21 +- .../token_in_token_out/test_mm_serde.py | 5 - .../test_error_sanitization.py | 1 - .../test_http_status_metrics.py | 1 - tests/entrypoints/serve/sagemaker/__init__.py | 0 tests/entrypoints/serve/sagemaker/conftest.py | 60 - .../test_sagemaker_handler_overrides.py | 629 - .../sagemaker/test_sagemaker_lora_adapters.py | 172 - .../test_sagemaker_middleware_integration.py | 347 - .../test_sagemaker_stateful_sessions.py | 154 - .../unit_tests/test_remote_vllm_server.py | 3 +- tests/evals/gpt_oss/README.md | 49 - tests/evals/gpt_oss/__init__.py | 2 - .../gpt_oss/configs/gpt-oss-20b-baseline.yaml | 6 - ...oss-20b-flashinfer-mxfp4-bf16-cutlass.yaml | 6 - ...-oss-20b-flashinfer-mxfp4-bf16-trtllm.yaml | 6 - ...ss-20b-flashinfer-mxfp4-mxfp8-cutlass.yaml | 6 - .../gpt_oss/configs/gpt-oss-20b-marlin.yaml | 6 - .../configs/gpt-oss-20b-rocm-baseline.yaml | 6 - ...t-oss-20b-rocm-quark-mxfp4-bf16-aiter.yaml | 8 - ...-oss-20b-rocm-quark-mxfp4-bf16-triton.yaml | 6 - ...t-oss-20b-rocm-quark-mxfp4-fp8-triton.yaml | 8 - ...t-oss-20b-sm100-fi-mxfp4-mxfp8-trtllm.yaml | 6 - .../gpt_oss/configs/gpt-oss-20b-sm120.yaml | 5 - .../configs/gpt-oss-20b-xpu-baseline.yaml | 5 - .../configs/gpt-oss-20b-xpu-triton-attn.yaml | 6 - tests/evals/gpt_oss/configs/models-b200.txt | 5 - tests/evals/gpt_oss/configs/models-gfx942.txt | 3 - tests/evals/gpt_oss/configs/models-gfx950.txt | 6 - tests/evals/gpt_oss/configs/models-h100.txt | 5 - tests/evals/gpt_oss/configs/models-spark.txt | 2 - tests/evals/gpt_oss/configs/models-xpu.txt | 3 - tests/evals/gpt_oss/conftest.py | 62 - tests/evals/gpt_oss/run_gpqa_eval.py | 34 - tests/evals/gpt_oss/test_gpqa_correctness.py | 175 - ..._fused_minimax_m3_qknorm_rope_kv_insert.py | 755 - .../model_loader/test_registry.py | 8 +- .../models/fixtures/mistral_small_3_chat.json | 1 - tests/models/language/pooling/embed_utils.py | 90 - .../generation/vlm_utils/model_utils.py | 274 - tests/models/qwen4_exp/test_config.py | 4 +- tests/parser/cohere/__init__.py | 2 - tests/parser/cohere/conftest.py | 18 - tests/parser/cohere/test_tool_calls.py | 288 - tests/parser/cohere/utils.py | 131 - tests/parser/mistral/__init__.py | 2 - .../__init__.py | 13 +- .../dummy_attention_backend.py | 4 +- tests/plugins_tests/test_endpoint_plugins.py | 35 +- tests/reasoning/utils.py | 62 - tests/renderers/test_token_offsets.py | 6 +- tests/test_zentorch_version_pin.py | 79 - tests/tokenizers_/test_detokenize.py | 427 - tests/tokenizers_/test_mistral.py | 2405 --- tests/tool_use/mistral/__init__.py | 0 tests/tool_use/mistral/conftest.py | 43 - .../mistral/test_mistral_tool_calls.py | 507 - tests/tool_use/mistral/utils.py | 46 - tests/tool_use/utils.py | 49 - tests/utils.py | 64 - tests/utils_/test_argparse_utils.py | 6 +- tests/v1/e2e/spec_decode/utils.py | 12 +- tests/v1/kv_connector/__init__.py | 0 .../__init__.py | 0 .../mooncake_integration/__init__.py | 0 .../config_sweep_accuracy_test.sh | 54 - .../mooncake_integration/run_accuracy_test.sh | 255 - .../mooncake_integration/test_accuracy.py | 81 - .../test_mooncake_imports.py | 22 - .../moriio_integration/__init__.py | 2 - .../moriio_integration/moriio_wide_dummy.py | 55 - .../kv_connector/nixl_integration/__init__.py | 2 - .../config_sweep_accuracy_test.sh | 106 - .../config_sweep_spec_decode_test.sh | 31 - .../nixl_side_channel_probe.py | 49 - .../nixl_integration/run_accuracy_test.sh | 329 - .../nixl_integration/run_edge_case_test.sh | 127 - .../run_mamba_prefix_cache_test.sh | 106 - .../run_multi_connector_accuracy_test.sh | 295 - .../run_multi_connector_edge_case_test.sh | 183 - .../run_xpu_disagg_accuracy_test.sh | 181 - .../spec_decode_acceptance_test.sh | 347 - .../nixl_integration/test_accuracy.py | 112 - .../nixl_integration/test_disagg_accuracy.py | 179 - .../nixl_integration/test_edge_cases.py | 80 - .../test_mamba_prefix_cache.py | 346 - .../test_multi_connector_edge_cases.py | 478 - .../nixl_integration/test_nixl_imports.py | 79 - .../nixl_integration/toy_proxy_server.py | 284 - .../config_sweep_accuracy_test.sh | 41 - tests/v1/kv_connector/unit/__init__.py | 0 .../unit/offloading_connector/__init__.py | 0 .../unit/offloading_connector/conftest.py | 7 - .../test_canonical_mapping.py | 479 - .../unit/offloading_connector/test_config.py | 1315 -- .../unit/offloading_connector/test_events.py | 811 - .../unit/offloading_connector/test_metrics.py | 1124 -- .../offloading_connector/test_scheduler.py | 5090 ----- .../test_worker_metadata.py | 53 - .../unit/offloading_connector/utils.py | 712 - .../unit/test_cache_pollution_prevention.py | 161 - tests/v1/kv_connector/unit/test_config.py | 209 - .../unit/test_error_propagation.py | 150 - .../unit/test_handshake_pp_aggregation.py | 233 - .../unit/test_hidden_states_connector.py | 230 - .../unit/test_invalid_blocks_correctness.py | 477 - .../unit/test_kv_connector_lifecycle.py | 77 - .../unit/test_kv_load_failure_recovery.py | 396 - .../kv_connector/unit/test_mooncake_proxy.py | 47 - .../unit/test_output_aggregator.py | 132 - .../unit/test_remote_decode_lifecycle.py | 260 - .../test_scheduler_kv_connector_override.py | 214 - .../unit/test_transfer_topology_sharded.py | 139 - tests/v1/kv_connector/unit/utils.py | 552 - tests/v1/kv_offload/cpu/test_manager.py | 4 +- tests/v1/kv_offload/test_factory.py | 112 +- tests/v1/kv_offload/test_file_mapper.py | 328 - tests/v1/kv_offload/tiering/__init__.py | 0 tests/v1/kv_offload/tiering/p2p/__init__.py | 0 tests/v1/kv_offload/tiering/p2p/conftest.py | 15 - .../tiering/p2p/p2p_connector_proxy.py | 418 - .../tiering/p2p/run_accuracy_test.sh | 343 - .../tiering/p2p/test_data_transport.py | 565 - .../v1/kv_offload/tiering/p2p/test_manager.py | 2303 --- .../kv_offload/tiering/p2p/test_sessions.py | 3509 ---- .../tiering/p2p/test_zmq_transport.py | 382 - .../kv_offload/tiering/test_async_lookup.py | 362 - .../kv_offload/tiering/test_backpressure.py | 807 - tests/v1/kv_offload/tiering/test_factory.py | 218 - tests/v1/kv_offload/tiering/test_fs_tier.py | 997 - tests/v1/kv_offload/tiering/test_kvcr_tier.py | 625 - tests/v1/kv_offload/tiering/test_metrics.py | 317 - tests/v1/kv_offload/tiering/test_obj_tier.py | 799 - .../tiering/test_tier_poll_thread.py | 490 - .../tiering/test_tiering_offloading.py | 1761 -- .../test_backend_xgrammar_lark.py | 146 - vllm/_custom_ops.py | 270 - vllm/config/model.py | 45 +- vllm/cute_utils/cvt.py | 192 - vllm/cute_utils/mbarrier.py | 47 - .../v1/example_hidden_states_connector.py | 635 - vllm/engine/arg_utils.py | 11 - vllm/entrypoints/anthropic/__init__.py | 2 - vllm/entrypoints/anthropic/api_router.py | 123 - vllm/entrypoints/anthropic/protocol.py | 451 - vllm/entrypoints/anthropic/serving.py | 1245 -- vllm/entrypoints/cli/launch.py | 115 - vllm/entrypoints/cli/main.py | 4 - vllm/entrypoints/cli/run_batch.py | 68 - vllm/entrypoints/cohere/__init__.py | 2 - vllm/entrypoints/cohere/api_router.py | 309 - .../entrypoints/cohere/cohere_chat_message.py | 144 - vllm/entrypoints/cohere/protocol.py | 389 - vllm/entrypoints/cohere/serving.py | 1824 -- vllm/entrypoints/common/offline.py | 6 +- vllm/entrypoints/generate/api_router.py | 80 +- vllm/entrypoints/generate/factories.py | 42 - .../generate/structured_decisions/__init__.py | 2 - .../structured_decisions/api_router.py | 86 - .../generate/structured_decisions/protocol.py | 83 - .../generate/structured_decisions/serving.py | 123 - .../launchers/api_server/app_state.py | 33 +- .../launchers/api_server/routers.py | 29 - vllm/entrypoints/launchers/app.py | 9 +- vllm/entrypoints/launchers/cli_args.py | 60 +- vllm/entrypoints/launchers/render/__init__.py | 0 .../entrypoints/launchers/render/app_state.py | 120 - vllm/entrypoints/launchers/render/entry.py | 129 - vllm/entrypoints/launchers/run_batch.py | 672 - vllm/entrypoints/llm.py | 3 +- vllm/entrypoints/openai/api_server.py | 4 - .../openai/chat_completion/protocol.py | 2 - vllm/entrypoints/openai/decisions/__init__.py | 2 - vllm/entrypoints/openai/decisions/adapters.py | 91 - .../openai/decisions/api_router.py | 56 - vllm/entrypoints/openai/decisions/protocol.py | 181 - .../openai/decisions/question_types.py | 240 - vllm/entrypoints/openai/decisions/serving.py | 102 - vllm/entrypoints/openai/decisions/state.py | 46 - .../openai/decisions/strategies.py | 1198 -- vllm/entrypoints/openai/run_batch.py | 37 - vllm/entrypoints/rl/__init__.py | 0 vllm/entrypoints/rl/factories.py | 11 - vllm/entrypoints/rl/offline.py | 68 - vllm/entrypoints/rl/online/__init__.py | 0 vllm/entrypoints/rl/online/api_router.py | 286 - vllm/entrypoints/rl/online/metrics.py | 56 - .../scale_out/derender/__init__.py | 0 .../scale_out/derender/api_router.py | 126 - .../entrypoints/scale_out/derender/serving.py | 434 - vllm/entrypoints/scale_out/factories.py | 42 +- vllm/entrypoints/scale_out/render/__init__.py | 2 - .../scale_out/render/api_router.py | 120 - vllm/entrypoints/scale_out/render/serving.py | 385 - .../token_in_token_out/api_router.py | 4 +- .../scale_out/token_in_token_out/protocol.py | 3 +- .../serve/elastic_ep/api_router.py | 118 - .../serve/instrumentator/__init__.py | 4 - .../serve/instrumentator/offline_docs.py | 50 - .../static/swagger-ui-bundle.js | 2 - .../instrumentator/static/swagger-ui.css | 3 - .../serve/middleware/authenticate.py | 2 +- vllm/entrypoints/serve/sagemaker/__init__.py | 4 - .../entrypoints/serve/sagemaker/api_router.py | 156 - .../speech_to_text/realtime/metrics.py | 78 - vllm/envs.py | 151 - .../nvidia_b200.json | 2025 -- .../nvidia_h100.json | 5185 ----- .../fused_qk_norm_rope/nvidia_b200.json | 2612 --- .../fused_qk_norm_rope/nvidia_h100.json | 2722 --- .../nvidia_b200.json | 1938 -- .../nvidia_h100.json | 1893 -- .../nvidia_b200.json | 2647 --- .../nvidia_h100.json | 3663 ---- .../rms_norm_per_block_quant/nvidia_b200.json | 2944 --- .../rms_norm_per_block_quant/nvidia_h100.json | 5017 ----- .../nvidia_b200.json | 2099 --- .../nvidia_h100.json | 2207 --- .../configs/silu_mul_fp8/nvidia_h100.json | 15711 ---------------- .../configs/silu_mul_fp8/nvidia_h200.json | 15711 ---------------- vllm/lora/ops/triton_ops/fp8_kernel_utils.py | 596 - ...VIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json | 146 - ...336,device_name=NVIDIA_A100-SXM4-80GB.json | 146 - ...VIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json | 218 - ...792,device_name=NVIDIA_A100-SXM4-80GB.json | 218 - ...VIDIA_H100_80GB_HBM3,dtype=int8_w8a16.json | 146 - ...VIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json | 218 - ...VIDIA_H100_80GB_HBM3,dtype=int8_w8a16.json | 146 - ...072,device_name=NVIDIA_H100_80GB_HBM3.json | 218 - ...ice_name=NVIDIA_H200,dtype=int8_w8a16.json | 146 - ...VIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json | 218 - ...584,device_name=NVIDIA_A100-SXM4-80GB.json | 218 - ...VIDIA_H100_80GB_HBM3,dtype=int8_w8a16.json | 146 - ...VIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json | 218 - ...168,device_name=NVIDIA_A100-SXM4-80GB.json | 218 - ...VIDIA_H100_80GB_HBM3,dtype=int8_w8a16.json | 146 - ...me=AMD_Instinct_MI300X,dtype=fp8_w8a8.json | 164 - ...=1024,device_name=AMD_Instinct_MI300X.json | 200 - ...me=AMD_Instinct_MI325X,dtype=fp8_w8a8.json | 164 - ...evice_name=NVIDIA_H100,dtype=fp8_w8a8.json | 123 - ...evice_name=NVIDIA_H200,dtype=fp8_w8a8.json | 146 - .../E=128,N=1024,device_name=NVIDIA_H200.json | 146 - .../E=128,N=1856,device_name=NVIDIA_B200.json | 139 - ...856,device_name=NVIDIA_H100_80GB_HBM3.json | 147 - ...evice_name=NVIDIA_H200,dtype=fp8_w8a8.json | 147 - .../E=128,N=1856,device_name=NVIDIA_L40S.json | 147 - ...192,device_name=NVIDIA_A100-SXM4-80GB.json | 146 - ...192,device_name=NVIDIA_H100_80GB_HBM3.json | 146 - ...E=128,N=192,device_name=NVIDIA_H20-3e.json | 146 - .../E=128,N=192,device_name=NVIDIA_H20.json | 146 - .../E=128,N=192,device_name=NVIDIA_H200.json | 146 - ...evice_name=NVIDIA_H800,dtype=fp8_w8a8.json | 147 - ...880,device_name=NVIDIA_H100_80GB_HBM3.json | 147 - ...=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json | 122 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 164 - ...ame=Intel(R)_Arc(TM)_Pro_B70_Graphics.json | 164 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...384,device_name=NVIDIA_H100_80GB_HBM3.json | 147 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...E=128,N=384,device_name=NVIDIA_H20-3e.json | 146 - .../E=128,N=384,device_name=NVIDIA_H20.json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - .../E=128,N=384,device_name=NVIDIA_H200.json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 165 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 147 - ...evice_name=NVIDIA_B200,dtype=fp8_w8a8.json | 147 - .../E=128,N=512,device_name=NVIDIA_B200.json | 147 - ...vice_name=NVIDIA_GB200,dtype=fp8_w8a8.json | 147 - ...512,device_name=NVIDIA_H100_80GB_HBM3.json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 147 - ...evice_name=NVIDIA_H200,dtype=fp8_w8a8.json | 146 - .../E=128,N=512,device_name=NVIDIA_H200.json | 147 - ...evice_name=NVIDIA_B200,dtype=fp8_w8a8.json | 146 - ...=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json | 114 - ...ll_Workstation_Edition,dtype=fp8_w8a8.json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 164 - ...N=768,device_name=AMD_Instinct_MI308X.json | 213 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - .../E=128,N=768,device_name=NVIDIA_B200.json | 147 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - .../E=128,N=768,device_name=NVIDIA_H20.json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - .../E=128,N=768,device_name=NVIDIA_H200.json | 146 - ...adeon_8060S_Graphics,dtype=int4_w4a16.json | 63 - ...name=NVIDIA_H100_80GB_HBM3,dtype=bf16.json | 82 - ...=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json | 82 - ...928,device_name=NVIDIA_H100_80GB_HBM3.json | 147 - .../E=128,N=928,device_name=NVIDIA_L40S.json | 147 - .../E=128,N=96,device_name=NVIDIA_H20.json | 146 - ...evice_name=NVIDIA_H200,dtype=fp8_w8a8.json | 147 - .../E=128,N=96,device_name=NVIDIA_H200.json | 147 - ...ll_Workstation_Edition,dtype=fp8_w8a8.json | 146 - ...=1024,device_name=AMD_Instinct_MI300X.json | 200 - ...evice_name=NVIDIA_B200,dtype=fp8_w8a8.json | 147 - .../E=16,N=1024,device_name=NVIDIA_B200.json | 146 - .../E=16,N=1024,device_name=NVIDIA_H100.json | 146 - ...evice_name=NVIDIA_H200,dtype=fp8_w8a8.json | 146 - .../E=16,N=1024,device_name=NVIDIA_H200.json | 146 - ...344,device_name=NVIDIA_A100-SXM4-40GB.json | 146 - ...344,device_name=NVIDIA_A100-SXM4-80GB.json | 146 - ...344,device_name=NVIDIA_H100_80GB_HBM3.json | 146 - ...VIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json | 146 - ...336,device_name=NVIDIA_A100-SXM4-80GB.json | 146 - ...VIDIA_H100_80GB_HBM3,dtype=int8_w8a16.json | 146 - ...VIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json | 218 - ...792,device_name=NVIDIA_A100-SXM4-80GB.json | 218 - ...VIDIA_H100_80GB_HBM3,dtype=int8_w8a16.json | 146 - ...792,device_name=NVIDIA_H100_80GB_HBM3.json | 146 - ...evice_name=NVIDIA_H200,dtype=fp8_w8a8.json | 146 - .../E=16,N=2048,device_name=NVIDIA_H200.json | 146 - ...688,device_name=NVIDIA_A100-SXM4-80GB.json | 146 - ...688,device_name=NVIDIA_H100_80GB_HBM3.json | 146 - ...VIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json | 146 - ...me=NVIDIA_H100_80GB_HBM3,dtype=float8.json | 146 - ...VIDIA_H100_80GB_HBM3,dtype=int8_w8a16.json | 146 - ...ice_name=NVIDIA_H200,dtype=int8_w8a16.json | 146 - ...=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json | 130 - ...VIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json | 146 - ...584,device_name=NVIDIA_A100-SXM4-80GB.json | 218 - ...VIDIA_H100_80GB_HBM3,dtype=int8_w8a16.json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 147 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 147 - ...=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json | 130 - ...VIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json | 146 - ...168,device_name=NVIDIA_A100-SXM4-80GB.json | 146 - ...me=NVIDIA_H100_80GB_HBM3,dtype=float8.json | 146 - ...VIDIA_H100_80GB_HBM3,dtype=int8_w8a16.json | 146 - ...=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json | 130 - ...N=192,device_name=AMD_Instinct_MI300X.json | 201 - ...AMD_Instinct_MI350_OAM,dtype=fp8_w8a8.json | 164 - ...192,device_name=NVIDIA_A800-SXM4-80GB.json | 146 - ...me=NVIDIA_B300_SXM6_AC,dtype=fp8_w8a8.json | 147 - ...E=160,N=192,device_name=NVIDIA_H20-3e.json | 146 - ...evice_name=NVIDIA_H200,dtype=fp8_w8a8.json | 147 - ...E=160,N=320,device_name=NVIDIA_H20-3e.json | 146 - ...me=AMD_Instinct_MI300X,dtype=fp8_w8a8.json | 164 - ...AMD_Instinct_MI350_OAM,dtype=fp8_w8a8.json | 164 - ...AMD_Instinct_MI355_OAM,dtype=fp8_w8a8.json | 164 - ...evice_name=NVIDIA_B200,dtype=fp8_w8a8.json | 163 - ...me=NVIDIA_B300_SXM6_AC,dtype=fp8_w8a8.json | 147 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...me=NVIDIA_B300_SXM6_AC,dtype=fp8_w8a8.json | 147 - ...ackwell_Server_Edition,dtype=fp8_w8a8.json | 147 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...Instinct_MI325X,block_shape=[128,128].json | 200 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 200 - ...dtype=int8_w8a8,block_shape=[128,128].json | 146 - ...NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8.json | 146 - ...dtype=int8_w8a8,block_shape=[128,128].json | 146 - ...NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8.json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 164 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 200 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 200 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...dtype=int8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - .../E=256,N=256,device_name=NVIDIA_H20.json | 147 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 147 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 147 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 200 - ...512,device_name=NVIDIA_A100_80GB_PCIe.json | 147 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 147 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 147 - ...512,device_name=NVIDIA_H100_80GB_HBM3.json | 146 - .../E=256,N=512,device_name=NVIDIA_H20.json | 147 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 147 - ...evice_name=NVIDIA_L40S,dtype=fp8_w8a8.json | 171 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 147 - ...=64,device_name=NVIDIA_A800-SXM4-80GB.json | 146 - .../E=32,N=1408,device_name=NVIDIA_B200.json | 147 - ...792,device_name=NVIDIA_H100_80GB_HBM3.json | 11 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 147 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 147 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...I350X,dtype=int4_w4a16,backend=flydsl.json | 114 - ...=AMD_Instinct_MI350X,dtype=int4_w4a16.json | 192 - ...0_OAM,dtype=int4_w4a16,backend=flydsl.json | 114 - ...D_Instinct_MI350_OAM,dtype=int4_w4a16.json | 192 - ...I355X,dtype=int4_w4a16,backend=flydsl.json | 114 - ...=AMD_Instinct_MI355X,dtype=int4_w4a16.json | 192 - ...5_OAM,dtype=int4_w4a16,backend=flydsl.json | 114 - ...D_Instinct_MI355_OAM,dtype=int4_w4a16.json | 192 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...I350X,dtype=int4_w4a16,backend=flydsl.json | 114 - ...0_OAM,dtype=int4_w4a16,backend=flydsl.json | 114 - ...I355X,dtype=int4_w4a16,backend=flydsl.json | 114 - ...5_OAM,dtype=int4_w4a16,backend=flydsl.json | 114 - ...evice_name=NVIDIA_B200,dtype=fp8_w8a8.json | 147 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...128,device_name=NVIDIA_A100-SXM4-80GB.json | 147 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 147 - .../E=512,N=128,device_name=NVIDIA_B200.json | 146 - ...vice_name=NVIDIA_GB200,dtype=fp8_w8a8.json | 147 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 43 - ...128,device_name=NVIDIA_H100_80GB_HBM3.json | 146 - ...E=512,N=128,device_name=NVIDIA_H20-3e.json | 146 - .../E=512,N=128,device_name=NVIDIA_H200.json | 146 - .../E=512,N=1344,device_name=NVIDIA_B200.json | 131 - ...00,dtype=fp8_w8a8,block_shape=[32,32].json | 115 - ...E=512,N=192,device_name=NVIDIA_H20-3e.json | 147 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 147 - .../E=512,N=256,device_name=NVIDIA_B200.json | 146 - ...vice_name=NVIDIA_GB200,dtype=fp8_w8a8.json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 147 - ...256,device_name=NVIDIA_H100_80GB_HBM3.json | 146 - ...E=512,N=256,device_name=NVIDIA_H20-3e.json | 146 - .../E=512,N=256,device_name=NVIDIA_H200.json | 146 - ...00,dtype=fp8_w8a8,block_shape=[64,64].json | 115 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 147 - .../E=512,N=512,device_name=NVIDIA_B200.json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 147 - ...vice_name=NVIDIA_GB200,dtype=fp8_w8a8.json | 146 - ...512,device_name=NVIDIA_H100_80GB_HBM3.json | 146 - ...E=512,N=512,device_name=NVIDIA_H20-3e.json | 146 - .../E=512,N=512,device_name=NVIDIA_H200.json | 146 - ...=64,device_name=NVIDIA_A100-SXM4-80GB.json | 147 - .../E=512,N=64,device_name=NVIDIA_B200.json | 146 - .../E=512,N=64,device_name=NVIDIA_H20-3e.json | 146 - .../E=512,N=64,device_name=NVIDIA_H200.json | 146 - .../E=512,N=672,device_name=NVIDIA_B200.json | 59 - ...=1408,device_name=AMD_Instinct_MI300X.json | 200 - ...N=176,device_name=AMD_Instinct_MI300X.json | 200 - ...N=352,device_name=AMD_Instinct_MI300X.json | 200 - ...N=704,device_name=AMD_Instinct_MI300X.json | 200 - ...N=128,device_name=AMD_Instinct_MI300X.json | 200 - ...N=256,device_name=AMD_Instinct_MI300X.json | 200 - ...256,device_name=NVIDIA_H100_80GB_HBM3.json | 146 - ...N=512,device_name=AMD_Instinct_MI300X.json | 200 - ...512,device_name=NVIDIA_H100_80GB_HBM3.json | 146 - ...ame=Intel(R)_Arc(TM)_Pro_B70_Graphics.json | 164 - ...280,device_name=NVIDIA_A100-SXM4-80GB.json | 146 - ...280,device_name=NVIDIA_A800-SXM4-80GB.json | 146 - ...=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json | 146 - ...280,device_name=NVIDIA_H100_80GB_HBM3.json | 146 - ...evice_name=NVIDIA_H200,dtype=fp8_w8a8.json | 146 - .../E=64,N=1280,device_name=NVIDIA_H200.json | 146 - .../E=64,N=1408,device_name=NVIDIA_B200.json | 147 - ...536,device_name=NVIDIA_H100_80GB_HBM3.json | 155 - ...device_name=NVIDIA_H20,dtype=fp8_w8a8.json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 147 - ...=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json | 146 - ...evice_name=NVIDIA_H200,dtype=fp8_w8a8.json | 146 - .../E=64,N=2560,device_name=NVIDIA_H200.json | 146 - ...device_name=NVIDIA_H20,dtype=fp8_w8a8.json | 146 - .../E=64,N=3072,device_name=NVIDIA_H20.json | 146 - ...=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json | 146 - ...320,device_name=NVIDIA_H100_80GB_HBM3.json | 146 - ...evice_name=NVIDIA_H200,dtype=fp8_w8a8.json | 146 - .../E=64,N=320,device_name=NVIDIA_H200.json | 146 - ...device_name=NVIDIA_H20,dtype=fp8_w8a8.json | 146 - .../E=64,N=384,device_name=NVIDIA_H20.json | 146 - ...ame=Intel(R)_Arc(TM)_Pro_B70_Graphics.json | 56 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 147 - ...640,device_name=NVIDIA_A100-SXM4-80GB.json | 146 - ...640,device_name=NVIDIA_A800-SXM4-80GB.json | 146 - ...VIDIA_GeForce_RTX_4090,dtype=fp8_w8a8.json | 146 - ...=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json | 146 - ...640,device_name=NVIDIA_H100_80GB_HBM3.json | 146 - ...evice_name=NVIDIA_H200,dtype=fp8_w8a8.json | 146 - .../E=64,N=640,device_name=NVIDIA_H200.json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 165 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 147 - ...device_name=NVIDIA_H20,dtype=fp8_w8a8.json | 146 - .../E=64,N=768,device_name=NVIDIA_H20.json | 146 - .../E=64,N=896,device_name=NVIDIA_H20.json | 146 - ...name=NVIDIA_H100_80GB_HBM3,dtype=bf16.json | 82 - ...=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json | 82 - ...N=192,device_name=AMD_Instinct_MI300X.json | 200 - ...N=384,device_name=AMD_Instinct_MI300X.json | 200 - ...384,device_name=NVIDIA_H100_80GB_HBM3.json | 146 - ...N=768,device_name=AMD_Instinct_MI300X.json | 200 - ...768,device_name=NVIDIA_H100_80GB_HBM3.json | 146 - ...me=AMD_Instinct_MI300X,dtype=fp8_w8a8.json | 164 - ...14336,device_name=AMD_Instinct_MI300X.json | 200 - ...me=AMD_Instinct_MI325X,dtype=fp8_w8a8.json | 164 - ...14336,device_name=AMD_Instinct_MI325X.json | 200 - ...=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json | 138 - ...evice_name=NVIDIA_H200,dtype=fp8_w8a8.json | 146 - .../E=8,N=14336,device_name=NVIDIA_H200.json | 146 - ...me=AMD_Instinct_MI300X,dtype=fp8_w8a8.json | 164 - ...16384,device_name=AMD_Instinct_MI300X.json | 200 - ...me=AMD_Instinct_MI325X,dtype=fp8_w8a8.json | 164 - ...16384,device_name=AMD_Instinct_MI325X.json | 200 - ...me=AMD_Instinct_MI300X,dtype=fp8_w8a8.json | 164 - ...=1792,device_name=AMD_Instinct_MI300X.json | 200 - ...me=AMD_Instinct_MI325X,dtype=fp8_w8a8.json | 164 - ...=1792,device_name=AMD_Instinct_MI325X.json | 200 - ...792,device_name=NVIDIA_A100-SXM4-40GB.json | 146 - ...792,device_name=NVIDIA_A100-SXM4-80GB.json | 146 - ...792,device_name=NVIDIA_H100_80GB_HBM3.json | 146 - ...evice_name=NVIDIA_H200,dtype=fp8_w8a8.json | 146 - .../E=8,N=1792,device_name=NVIDIA_H200.json | 146 - ...me=AMD_Instinct_MI300X,dtype=fp8_w8a8.json | 164 - ...=2048,device_name=AMD_Instinct_MI300X.json | 200 - ...me=AMD_Instinct_MI325X,dtype=fp8_w8a8.json | 164 - ...=2048,device_name=AMD_Instinct_MI325X.json | 200 - ...048,device_name=NVIDIA_A100-SXM4-80GB.json | 146 - ...=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json | 146 - ...048,device_name=NVIDIA_H100_80GB_HBM3.json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 154 - ...evice_name=NVIDIA_H200,dtype=fp8_w8a8.json | 146 - .../E=8,N=2048,device_name=NVIDIA_H200.json | 146 - ...me=AMD_Instinct_MI300X,dtype=fp8_w8a8.json | 164 - ...=3584,device_name=AMD_Instinct_MI300X.json | 200 - ...me=AMD_Instinct_MI325X,dtype=fp8_w8a8.json | 164 - ...=3584,device_name=AMD_Instinct_MI325X.json | 200 - ...584,device_name=NVIDIA_A100-SXM4-40GB.json | 146 - ...584,device_name=NVIDIA_A100-SXM4-80GB.json | 146 - ...VIDIA_GeForce_RTX_4090,dtype=fp8_w8a8.json | 146 - ...=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json | 146 - ...584,device_name=NVIDIA_H100_80GB_HBM3.json | 146 - ...evice_name=NVIDIA_H200,dtype=fp8_w8a8.json | 146 - .../E=8,N=3584,device_name=NVIDIA_H200.json | 146 - .../E=8,N=3584,device_name=NVIDIA_L40S.json | 173 - ...me=AMD_Instinct_MI300X,dtype=fp8_w8a8.json | 164 - ...=4096,device_name=AMD_Instinct_MI300X.json | 200 - ...me=AMD_Instinct_MI325X,dtype=fp8_w8a8.json | 164 - ...=4096,device_name=AMD_Instinct_MI325X.json | 200 - ...096,device_name=NVIDIA_A100-SXM4-80GB.json | 146 - ...=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json | 146 - ...096,device_name=NVIDIA_H100_80GB_HBM3.json | 146 - ...evice_name=NVIDIA_H200,dtype=fp8_w8a8.json | 146 - .../E=8,N=4096,device_name=NVIDIA_H200.json | 146 - ...me=AMD_Instinct_MI300X,dtype=fp8_w8a8.json | 164 - ...=7168,device_name=AMD_Instinct_MI300X.json | 200 - ...me=AMD_Instinct_MI325X,dtype=fp8_w8a8.json | 164 - ...=7168,device_name=AMD_Instinct_MI325X.json | 200 - ...168,device_name=NVIDIA_A100-SXM4-80GB.json | 146 - ...=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json | 146 - ...168,device_name=NVIDIA_H100_80GB_HBM3.json | 146 - ...evice_name=NVIDIA_H200,dtype=fp8_w8a8.json | 146 - .../E=8,N=7168,device_name=NVIDIA_H200.json | 146 - ...me=AMD_Instinct_MI300X,dtype=fp8_w8a8.json | 164 - ...=8192,device_name=AMD_Instinct_MI300X.json | 200 - ...me=AMD_Instinct_MI325X,dtype=fp8_w8a8.json | 164 - ...=8192,device_name=AMD_Instinct_MI325X.json | 200 - ...=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json | 146 - ...evice_name=NVIDIA_H200,dtype=fp8_w8a8.json | 146 - .../layers/fused_moe/configs/README | 12 - vllm/model_executor/layers/linear.py | 2 - ..._Pro_B60_Graphics,cache_dtype=float16.json | 51 - ..._Pro_B60_Graphics,cache_dtype=float32.json | 51 - ..._Pro_B70_Graphics,cache_dtype=float16.json | 51 - ..._Pro_B70_Graphics,cache_dtype=float32.json | 51 - ...D_Instinct_MI300X,cache_dtype=float16.json | 51 - ...D_Instinct_MI300X,cache_dtype=float32.json | 51 - ...D_Instinct_MI325X,cache_dtype=float16.json | 51 - ...D_Instinct_MI325X,cache_dtype=float32.json | 51 - ...nstinct_MI350_OAM,cache_dtype=float16.json | 51 - ...nstinct_MI350_OAM,cache_dtype=float32.json | 51 - ...nstinct_MI355_OAM,cache_dtype=float16.json | 51 - ...nstinct_MI355_OAM,cache_dtype=float32.json | 51 - ..._Pro_B60_Graphics,cache_dtype=float16.json | 63 - ..._Pro_B60_Graphics,cache_dtype=float32.json | 63 - ..._Pro_B70_Graphics,cache_dtype=float16.json | 63 - ..._Pro_B70_Graphics,cache_dtype=float32.json | 63 - ..._name=NVIDIA_B200,cache_dtype=float16.json | 87 - ..._name=NVIDIA_B200,cache_dtype=float32.json | 87 - ...name=NVIDIA_GB200,cache_dtype=float16.json | 87 - ...name=NVIDIA_GB200,cache_dtype=float32.json | 87 - ...IA_H100_80GB_HBM3,cache_dtype=float16.json | 87 - ...IA_H100_80GB_HBM3,cache_dtype=float32.json | 87 - ..._name=NVIDIA_H200,cache_dtype=float16.json | 87 - ..._name=NVIDIA_H200,cache_dtype=float32.json | 87 - ...ll_Server_Edition,cache_dtype=float16.json | 87 - ...ll_Server_Edition,cache_dtype=float32.json | 87 - ..._Pro_B60_Graphics,cache_dtype=float16.json | 63 - ..._Pro_B60_Graphics,cache_dtype=float32.json | 63 - ..._Pro_B70_Graphics,cache_dtype=float16.json | 47 - ..._Pro_B70_Graphics,cache_dtype=float32.json | 39 - ..._Pro_B60_Graphics,cache_dtype=float16.json | 63 - ..._Pro_B60_Graphics,cache_dtype=float32.json | 63 - ..._Pro_B70_Graphics,cache_dtype=float16.json | 47 - ..._Pro_B70_Graphics,cache_dtype=float32.json | 47 - .../layers/quantization/__init__.py | 10 - .../layers/quantization/fbgemm_fp8.py | 186 - .../layers/quantization/fp_quant.py | 400 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 164 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 164 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 164 - ...dtype=int8_w8a8,block_shape=[128,128].json | 146 - ...dtype=int8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 26 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 164 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 164 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 164 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 82 - ...dtype=int8_w8a8,block_shape=[128,128].json | 146 - ...dtype=int8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 26 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 26 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 82 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 82 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 164 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 164 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 164 - ...dtype=int8_w8a8,block_shape=[128,128].json | 146 - ...dtype=int8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 26 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 82 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 82 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 164 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 164 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 164 - ...dtype=int8_w8a8,block_shape=[128,128].json | 146 - ...dtype=int8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 26 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 82 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 164 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 164 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 164 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 82 - ...dtype=int8_w8a8,block_shape=[128,128].json | 146 - ...dtype=int8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...dtype=int8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 26 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 26 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 164 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 164 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 164 - ...dtype=int8_w8a8,block_shape=[128,128].json | 146 - ...dtype=int8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 26 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 82 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 164 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 164 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 164 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 82 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...dtype=int8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 26 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 82 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 164 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 164 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 164 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 82 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...dtype=int8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 26 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 164 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 164 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 164 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 82 - ...dtype=int8_w8a8,block_shape=[128,128].json | 146 - ...dtype=int8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...dtype=int8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 26 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 26 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 164 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 164 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 164 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 82 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 82 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 82 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 164 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 164 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 164 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 82 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...dtype=int8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 26 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 164 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 164 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 164 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 82 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...dtype=int8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 26 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 164 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 164 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 164 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 82 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...dtype=int8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 26 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 82 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 164 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 164 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 164 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 82 - ...dtype=int8_w8a8,block_shape=[128,128].json | 146 - ...dtype=int8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...dtype=int8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 18 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 26 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 82 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 164 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 164 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 164 - ...dtype=int8_w8a8,block_shape=[128,128].json | 146 - ...dtype=int8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 26 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 164 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 164 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 164 - ...dtype=int8_w8a8,block_shape=[128,128].json | 146 - ...dtype=int8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 26 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 164 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 164 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 164 - ...dtype=int8_w8a8,block_shape=[128,128].json | 146 - ...dtype=int8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 26 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 164 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 164 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 164 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 82 - ...dtype=int8_w8a8,block_shape=[128,128].json | 146 - ...dtype=int8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...dtype=int8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 26 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 26 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 164 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 164 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 164 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 82 - ...dtype=int8_w8a8,block_shape=[128,128].json | 146 - ...dtype=int8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...dtype=int8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 26 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 26 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 164 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 164 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 164 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 82 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...dtype=int8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 26 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 164 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 164 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 164 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 82 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...dtype=int8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 26 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 164 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 164 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 164 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 82 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...dtype=int8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 26 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 164 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 164 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 164 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 164 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 164 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 164 - ...,dtype=fp8_w8a8,block_shape=[128,128].json | 146 - .../quantization/utils/configs/README.md | 3 - .../layers/rotary_embedding/__init__.py | 150 - .../layers/rotary_embedding/bailing_mrope.py | 351 - .../dynamic_ntk_alpha_rope.py | 43 - .../dynamic_ntk_scaling_rope.py | 73 - .../layers/rotary_embedding/fope.py | 199 - .../layers/rotary_embedding/gemma4_rope.py | 84 - .../rotary_embedding/linear_scaling_rope.py | 115 - .../layers/rotary_embedding/llama3_rope.py | 54 - .../rotary_embedding/llama4_vision_rope.py | 83 - .../rotary_embedding/ntk_scaling_rope.py | 47 - .../phi3_long_rope_scaled_rope.py | 159 - .../model_loader/default_loader.py | 2 +- vllm/model_executor/models/registry.py | 9 + vllm/model_executor/models/vision.py | 63 +- vllm/multimodal/processing/context.py | 4 - vllm/parser/abstract_parser.py | 1 - vllm/parser/mistral.py | 1002 - vllm/parser/parser_manager.py | 42 - vllm/platforms/interface.py | 13 - vllm/platforms/rocm.py | 4 - vllm/plugins/__init__.py | 2 - vllm/plugins/endpoint_plugins/interface.py | 30 +- vllm/plugins/io_processors/__init__.py | 88 - vllm/plugins/io_processors/interface.py | 80 - vllm/pooling_params.py | 4 +- vllm/renderers/cohere.py | 829 - vllm/renderers/mistral.py | 116 - vllm/renderers/online_derenderer.py | 1265 -- vllm/renderers/online_renderer.py | 60 +- vllm/renderers/registry.py | 9 - vllm/sampling_params.py | 63 +- vllm/tasks.py | 5 +- vllm/tokenizers/mistral.py | 564 - vllm/tokenizers/registry.py | 41 +- vllm/tool_parsers/streaming.py | 24 +- .../chat_templates/registry.py | 26 +- .../chat_templates/template_basic.jinja | 3 - .../chat_templates/template_blip2.jinja | 11 - .../chat_templates/template_chatml.jinja | 10 - .../template_deepseek_ocr.jinja | 14 - .../template_deepseek_vl2.jinja | 23 - .../chat_templates/template_kimi_audio.jinja | 13 - .../chat_templates/template_minicpmv45.jinja | 93 - .../configs/mimo_v2_omni.py | 65 - vllm/transformers_utils/processor.py | 27 +- .../transformers_utils/processors/__init__.py | 57 - vllm/triton_utils/dispatcher.py | 175 - vllm/utils/mistral.py | 38 - vllm/v1/attention/ops/paged_attn.py | 53 - vllm/v1/kv_offload/file_mapper.py | 144 - vllm/v1/kv_offload/tiering/__init__.py | 0 vllm/v1/kv_offload/tiering/async_lookup.py | 290 - vllm/v1/kv_offload/tiering/backpressure.py | 393 - vllm/v1/kv_offload/tiering/base.py | 423 - .../v1/kv_offload/tiering/example/__init__.py | 0 vllm/v1/kv_offload/tiering/example/manager.py | 205 - vllm/v1/kv_offload/tiering/factory.py | 174 - vllm/v1/kv_offload/tiering/fs/__init__.py | 0 vllm/v1/kv_offload/tiering/fs/io.py | 218 - vllm/v1/kv_offload/tiering/fs/manager.py | 353 - vllm/v1/kv_offload/tiering/fs/thread_pool.py | 194 - vllm/v1/kv_offload/tiering/kvcr/__init__.py | 2 - vllm/v1/kv_offload/tiering/kvcr/manager.py | 655 - vllm/v1/kv_offload/tiering/manager.py | 1282 -- vllm/v1/kv_offload/tiering/metrics.py | 320 - vllm/v1/kv_offload/tiering/obj/__init__.py | 0 vllm/v1/kv_offload/tiering/obj/config.py | 46 - vllm/v1/kv_offload/tiering/obj/manager.py | 461 - vllm/v1/kv_offload/tiering/p2p/__init__.py | 0 .../tiering/p2p/control/__init__.py | 17 - .../v1/kv_offload/tiering/p2p/control/base.py | 176 - vllm/v1/kv_offload/tiering/p2p/control/zmq.py | 375 - .../kv_offload/tiering/p2p/data/__init__.py | 10 - vllm/v1/kv_offload/tiering/p2p/data/base.py | 325 - vllm/v1/kv_offload/tiering/p2p/data/nixl.py | 523 - vllm/v1/kv_offload/tiering/p2p/manager.py | 1056 -- .../tiering/p2p/session/__init__.py | 17 - .../kv_offload/tiering/p2p/session/client.py | 548 - .../tiering/p2p/session/protocol.py | 341 - .../kv_offload/tiering/p2p/session/server.py | 1281 -- .../kv_offload/tiering/p2p/session/session.py | 666 - vllm/v1/kv_offload/tiering/spec.py | 579 - vllm/v1/metrics/perf.py | 6 - vllm/v1/structured_output/backend_guidance.py | 6 +- vllm/v1/structured_output/backend_xgrammar.py | 21 +- vllm/v1/worker/worker_base.py | 6 - 1086 files changed, 122 insertions(+), 291919 deletions(-) delete mode 100644 cmake/cpu_extension.cmake delete mode 100644 csrc/cpu/activation.cpp delete mode 100644 csrc/cpu/activation_lut_bf16.cpp delete mode 100644 csrc/cpu/cpu_arch_macros.h delete mode 100644 csrc/cpu/cpu_attn.cpp delete mode 100644 csrc/cpu/cpu_attn_amx.hpp delete mode 100644 csrc/cpu/cpu_attn_amx_fp8.hpp delete mode 100644 csrc/cpu/cpu_attn_fp8.hpp delete mode 100644 csrc/cpu/cpu_attn_impl.hpp delete mode 100644 csrc/cpu/cpu_attn_neon.hpp delete mode 100644 csrc/cpu/cpu_attn_neon_bfmmla.hpp delete mode 100644 csrc/cpu/cpu_attn_rvv.hpp delete mode 100644 csrc/cpu/cpu_attn_vec.hpp delete mode 100644 csrc/cpu/cpu_attn_vec16.hpp delete mode 100644 csrc/cpu/cpu_attn_vsx.hpp delete mode 100644 csrc/cpu/cpu_attn_vxe.hpp delete mode 100644 csrc/cpu/cpu_fused_moe.cpp delete mode 100644 csrc/cpu/cpu_fused_moe_activations.hpp delete mode 100644 csrc/cpu/cpu_fused_moe_int8.cpp delete mode 100644 csrc/cpu/cpu_isa.cpp delete mode 100644 csrc/cpu/cpu_tanhf_neon.hpp delete mode 100644 csrc/cpu/cpu_types.hpp delete mode 100644 csrc/cpu/cpu_types_arm.hpp delete mode 100644 csrc/cpu/cpu_types_riscv.hpp delete mode 100644 csrc/cpu/cpu_types_riscv_defs.hpp delete mode 100644 csrc/cpu/cpu_types_riscv_impl.hpp delete mode 100644 csrc/cpu/cpu_types_scalar.hpp delete mode 100644 csrc/cpu/cpu_types_vsx.hpp delete mode 100644 csrc/cpu/cpu_types_vxe.hpp delete mode 100644 csrc/cpu/cpu_types_x86.hpp delete mode 100644 csrc/cpu/cpu_wna16.cpp delete mode 100644 csrc/cpu/dnnl_helper.cpp delete mode 100644 csrc/cpu/dnnl_helper.h delete mode 100644 csrc/cpu/dnnl_kernels.cpp delete mode 100644 csrc/cpu/dynamic_4bit_int_moe_cpu.cpp delete mode 100644 csrc/cpu/float_convert.hpp delete mode 100644 csrc/cpu/generate_cpu_attn_dispatch.py delete mode 100644 csrc/cpu/layernorm.cpp delete mode 100644 csrc/cpu/mamba_cpu.cpp delete mode 100644 csrc/cpu/mamba_kernels.hpp delete mode 100644 csrc/cpu/micro_gemm/cpu_micro_gemm_amx.hpp delete mode 100644 csrc/cpu/micro_gemm/cpu_micro_gemm_impl.hpp delete mode 100644 csrc/cpu/micro_gemm/cpu_micro_gemm_int8_neon.hpp delete mode 100644 csrc/cpu/micro_gemm/cpu_micro_gemm_int8_vsx.hpp delete mode 100644 csrc/cpu/micro_gemm/cpu_micro_gemm_neon.hpp delete mode 100644 csrc/cpu/micro_gemm/cpu_micro_gemm_rvv.hpp delete mode 100644 csrc/cpu/micro_gemm/cpu_micro_gemm_vec.hpp delete mode 100644 csrc/cpu/micro_gemm/cpu_micro_gemm_vsx.hpp delete mode 100644 csrc/cpu/mla_decode.cpp delete mode 100644 csrc/cpu/ops.h delete mode 100644 csrc/cpu/pos_encoding.cpp delete mode 100644 csrc/cpu/sampling_kernels.cpp delete mode 100644 csrc/cpu/sgl-kernels/blas_gemm.h delete mode 100644 csrc/cpu/sgl-kernels/bmm.cpp delete mode 100644 csrc/cpu/sgl-kernels/common.h delete mode 100644 csrc/cpu/sgl-kernels/compressor.cpp delete mode 100644 csrc/cpu/sgl-kernels/conv.cpp delete mode 100644 csrc/cpu/sgl-kernels/conv_arm.h delete mode 100644 csrc/cpu/sgl-kernels/decode.cpp delete mode 100644 csrc/cpu/sgl-kernels/extend.cpp delete mode 100644 csrc/cpu/sgl-kernels/fla.cpp delete mode 100644 csrc/cpu/sgl-kernels/flash_attn.h delete mode 100644 csrc/cpu/sgl-kernels/flash_mla.cpp delete mode 100644 csrc/cpu/sgl-kernels/gemm.cpp delete mode 100644 csrc/cpu/sgl-kernels/gemm.h delete mode 100644 csrc/cpu/sgl-kernels/gemm_fp8.cpp delete mode 100644 csrc/cpu/sgl-kernels/gemm_fp8_w8a8.cpp delete mode 100644 csrc/cpu/sgl-kernels/gemm_int4.cpp delete mode 100644 csrc/cpu/sgl-kernels/gemm_int8.cpp delete mode 100644 csrc/cpu/sgl-kernels/indexer.cpp delete mode 100644 csrc/cpu/sgl-kernels/mhc.cpp delete mode 100644 csrc/cpu/sgl-kernels/mla_cache.cpp delete mode 100644 csrc/cpu/sgl-kernels/moe.cpp delete mode 100644 csrc/cpu/sgl-kernels/moe.h delete mode 100644 csrc/cpu/sgl-kernels/moe_fp8.cpp delete mode 100644 csrc/cpu/sgl-kernels/moe_fp8_w8a8.cpp delete mode 100644 csrc/cpu/sgl-kernels/moe_int4.cpp delete mode 100644 csrc/cpu/sgl-kernels/moe_int8.cpp delete mode 100644 csrc/cpu/sgl-kernels/paged_mqa_logits.cpp delete mode 100644 csrc/cpu/sgl-kernels/store_cache.cpp delete mode 100644 csrc/cpu/sgl-kernels/topk.cpp delete mode 100644 csrc/cpu/sgl-kernels/vec.h delete mode 100644 csrc/cpu/sgl-kernels/vec_pack.h delete mode 100644 csrc/cpu/shm.cpp delete mode 100644 csrc/cpu/spec_decode_utils.cpp delete mode 100644 csrc/cpu/torch_bindings.cpp delete mode 100644 csrc/cpu/utils.cpp delete mode 100644 csrc/cpu/utils.hpp delete mode 100644 csrc/dsv3_fused_a_gemm.cu delete mode 100644 csrc/fused_deepseek_v4_qnorm_rope_kv_insert_kernel.cu delete mode 100644 csrc/fused_kimi_k3_mla_key_concat_kv_cache_kernel.cu delete mode 100644 csrc/fused_minimax_m3_qknorm_rope_kv_insert_kernel.cu delete mode 100644 csrc/fused_muse_spark_qknorm_rope_kv_insert_kernel.cu delete mode 100644 csrc/fused_qknorm_rope_cache_kernel.cu delete mode 100644 csrc/kimi_k3/attn_res_kernel.cu delete mode 100644 csrc/minimax_reduce_rms_kernel.cu delete mode 100644 csrc/minimax_reduce_rms_kernel.h delete mode 100644 csrc/muse_spark_mul_tanh_soft_cap_kernel.cu delete mode 100644 requirements/cpu.txt delete mode 100644 requirements/test/cpu.txt delete mode 100644 tests/entrypoints/anthropic/__init__.py delete mode 100644 tests/entrypoints/anthropic/test_anthropic_messages_conversion.py delete mode 100644 tests/entrypoints/anthropic/test_messages.py delete mode 100644 tests/entrypoints/anthropic/test_protocol_exports.py delete mode 100644 tests/entrypoints/cohere/__init__.py delete mode 100644 tests/entrypoints/cohere/test_api_router.py delete mode 100644 tests/entrypoints/cohere/test_chat_v2.py delete mode 100644 tests/entrypoints/cohere/test_cohere_chat_message.py delete mode 100644 tests/entrypoints/cohere/test_protocol.py delete mode 100644 tests/entrypoints/cohere/test_registry_and_args.py delete mode 100644 tests/entrypoints/cohere/test_serving_conversion.py delete mode 100644 tests/entrypoints/cohere/test_serving_streaming.py delete mode 100644 tests/entrypoints/generate/structured_decisions/__init__.py delete mode 100644 tests/entrypoints/generate/structured_decisions/test_decisions.py delete mode 100644 tests/entrypoints/generate/structured_decisions/test_question_types.py delete mode 100644 tests/entrypoints/generate/structured_decisions/test_structured_decisions_e2e.py delete mode 100644 tests/entrypoints/launchers/render/__init__.py delete mode 100644 tests/entrypoints/launchers/render/test_app_state.py delete mode 100644 tests/entrypoints/launchers/test_run_batch.py delete mode 100644 tests/entrypoints/openai/decisions/test_decisions_e2e.py delete mode 100644 tests/entrypoints/rl/__init__.py delete mode 100644 tests/entrypoints/rl/conftest.py delete mode 100644 tests/entrypoints/rl/return_routed_experts/__init__.py delete mode 100644 tests/entrypoints/rl/return_routed_experts/test_openai_api.py delete mode 100644 tests/entrypoints/rl/return_routed_experts/test_route_request_bodies.py delete mode 100644 tests/entrypoints/rl/return_routed_experts/test_scale_out_api.py delete mode 100644 tests/entrypoints/rl/state_transitions/__init__.py delete mode 100644 tests/entrypoints/rl/state_transitions/test_pause_resume.py delete mode 100644 tests/entrypoints/rl/state_transitions/test_sleep.py delete mode 100644 tests/entrypoints/rl/weight_transfer/__init__.py delete mode 100644 tests/entrypoints/rl/weight_transfer/test_offline.py delete mode 100644 tests/entrypoints/rl/weight_transfer_metrics/__init__.py delete mode 100644 tests/entrypoints/rl/weight_transfer_metrics/test_weight_operation_metrics.py delete mode 100644 tests/entrypoints/scale_out/derender/__init__.py delete mode 100644 tests/entrypoints/scale_out/derender/test_derender_parity.py delete mode 100644 tests/entrypoints/scale_out/derender/utils.py delete mode 100644 tests/entrypoints/scale_out/ec_integration/README.md delete mode 100755 tests/entrypoints/scale_out/ec_integration/run_scale_out_ec_e2e_test.sh delete mode 100644 tests/entrypoints/scale_out/ec_integration/test_scale_out_ec_e2e.py delete mode 100644 tests/entrypoints/scale_out/render/__init__.py delete mode 100644 tests/entrypoints/scale_out/render/test_launch_render.py delete mode 100644 tests/entrypoints/scale_out/render/test_render.py delete mode 100644 tests/entrypoints/scale_out/render/test_render_multimodal.py delete mode 100644 tests/entrypoints/serve/sagemaker/__init__.py delete mode 100644 tests/entrypoints/serve/sagemaker/conftest.py delete mode 100644 tests/entrypoints/serve/sagemaker/test_sagemaker_handler_overrides.py delete mode 100644 tests/entrypoints/serve/sagemaker/test_sagemaker_lora_adapters.py delete mode 100644 tests/entrypoints/serve/sagemaker/test_sagemaker_middleware_integration.py delete mode 100644 tests/entrypoints/serve/sagemaker/test_sagemaker_stateful_sessions.py delete mode 100644 tests/evals/gpt_oss/README.md delete mode 100644 tests/evals/gpt_oss/__init__.py delete mode 100644 tests/evals/gpt_oss/configs/gpt-oss-20b-baseline.yaml delete mode 100644 tests/evals/gpt_oss/configs/gpt-oss-20b-flashinfer-mxfp4-bf16-cutlass.yaml delete mode 100644 tests/evals/gpt_oss/configs/gpt-oss-20b-flashinfer-mxfp4-bf16-trtllm.yaml delete mode 100644 tests/evals/gpt_oss/configs/gpt-oss-20b-flashinfer-mxfp4-mxfp8-cutlass.yaml delete mode 100644 tests/evals/gpt_oss/configs/gpt-oss-20b-marlin.yaml delete mode 100644 tests/evals/gpt_oss/configs/gpt-oss-20b-rocm-baseline.yaml delete mode 100644 tests/evals/gpt_oss/configs/gpt-oss-20b-rocm-quark-mxfp4-bf16-aiter.yaml delete mode 100644 tests/evals/gpt_oss/configs/gpt-oss-20b-rocm-quark-mxfp4-bf16-triton.yaml delete mode 100644 tests/evals/gpt_oss/configs/gpt-oss-20b-rocm-quark-mxfp4-fp8-triton.yaml delete mode 100644 tests/evals/gpt_oss/configs/gpt-oss-20b-sm100-fi-mxfp4-mxfp8-trtllm.yaml delete mode 100644 tests/evals/gpt_oss/configs/gpt-oss-20b-sm120.yaml delete mode 100644 tests/evals/gpt_oss/configs/gpt-oss-20b-xpu-baseline.yaml delete mode 100644 tests/evals/gpt_oss/configs/gpt-oss-20b-xpu-triton-attn.yaml delete mode 100644 tests/evals/gpt_oss/configs/models-b200.txt delete mode 100644 tests/evals/gpt_oss/configs/models-gfx942.txt delete mode 100644 tests/evals/gpt_oss/configs/models-gfx950.txt delete mode 100644 tests/evals/gpt_oss/configs/models-h100.txt delete mode 100644 tests/evals/gpt_oss/configs/models-spark.txt delete mode 100644 tests/evals/gpt_oss/configs/models-xpu.txt delete mode 100644 tests/evals/gpt_oss/conftest.py delete mode 100644 tests/evals/gpt_oss/run_gpqa_eval.py delete mode 100644 tests/evals/gpt_oss/test_gpqa_correctness.py delete mode 100644 tests/kernels/test_fused_minimax_m3_qknorm_rope_kv_insert.py delete mode 100644 tests/models/fixtures/mistral_small_3_chat.json delete mode 100644 tests/models/language/pooling/embed_utils.py delete mode 100644 tests/parser/cohere/__init__.py delete mode 100644 tests/parser/cohere/conftest.py delete mode 100644 tests/parser/cohere/test_tool_calls.py delete mode 100644 tests/parser/cohere/utils.py delete mode 100644 tests/parser/mistral/__init__.py delete mode 100644 tests/test_zentorch_version_pin.py delete mode 100644 tests/tokenizers_/test_detokenize.py delete mode 100644 tests/tokenizers_/test_mistral.py delete mode 100644 tests/tool_use/mistral/__init__.py delete mode 100644 tests/tool_use/mistral/conftest.py delete mode 100644 tests/tool_use/mistral/test_mistral_tool_calls.py delete mode 100644 tests/tool_use/mistral/utils.py delete mode 100644 tests/v1/kv_connector/__init__.py delete mode 100644 tests/v1/kv_connector/extract_hidden_states_integration/__init__.py delete mode 100644 tests/v1/kv_connector/mooncake_integration/__init__.py delete mode 100755 tests/v1/kv_connector/mooncake_integration/config_sweep_accuracy_test.sh delete mode 100755 tests/v1/kv_connector/mooncake_integration/run_accuracy_test.sh delete mode 100644 tests/v1/kv_connector/mooncake_integration/test_accuracy.py delete mode 100644 tests/v1/kv_connector/mooncake_integration/test_mooncake_imports.py delete mode 100644 tests/v1/kv_connector/moriio_integration/__init__.py delete mode 100644 tests/v1/kv_connector/moriio_integration/moriio_wide_dummy.py delete mode 100644 tests/v1/kv_connector/nixl_integration/__init__.py delete mode 100755 tests/v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh delete mode 100755 tests/v1/kv_connector/nixl_integration/config_sweep_spec_decode_test.sh delete mode 100644 tests/v1/kv_connector/nixl_integration/nixl_side_channel_probe.py delete mode 100755 tests/v1/kv_connector/nixl_integration/run_accuracy_test.sh delete mode 100755 tests/v1/kv_connector/nixl_integration/run_edge_case_test.sh delete mode 100755 tests/v1/kv_connector/nixl_integration/run_mamba_prefix_cache_test.sh delete mode 100755 tests/v1/kv_connector/nixl_integration/run_multi_connector_accuracy_test.sh delete mode 100755 tests/v1/kv_connector/nixl_integration/run_multi_connector_edge_case_test.sh delete mode 100644 tests/v1/kv_connector/nixl_integration/run_xpu_disagg_accuracy_test.sh delete mode 100755 tests/v1/kv_connector/nixl_integration/spec_decode_acceptance_test.sh delete mode 100644 tests/v1/kv_connector/nixl_integration/test_accuracy.py delete mode 100644 tests/v1/kv_connector/nixl_integration/test_disagg_accuracy.py delete mode 100644 tests/v1/kv_connector/nixl_integration/test_edge_cases.py delete mode 100644 tests/v1/kv_connector/nixl_integration/test_mamba_prefix_cache.py delete mode 100644 tests/v1/kv_connector/nixl_integration/test_multi_connector_edge_cases.py delete mode 100644 tests/v1/kv_connector/nixl_integration/test_nixl_imports.py delete mode 100644 tests/v1/kv_connector/nixl_integration/toy_proxy_server.py delete mode 100755 tests/v1/kv_connector/nixl_push_integration/config_sweep_accuracy_test.sh delete mode 100644 tests/v1/kv_connector/unit/__init__.py delete mode 100644 tests/v1/kv_connector/unit/offloading_connector/__init__.py delete mode 100644 tests/v1/kv_connector/unit/offloading_connector/conftest.py delete mode 100644 tests/v1/kv_connector/unit/offloading_connector/test_canonical_mapping.py delete mode 100644 tests/v1/kv_connector/unit/offloading_connector/test_config.py delete mode 100644 tests/v1/kv_connector/unit/offloading_connector/test_events.py delete mode 100644 tests/v1/kv_connector/unit/offloading_connector/test_metrics.py delete mode 100644 tests/v1/kv_connector/unit/offloading_connector/test_scheduler.py delete mode 100644 tests/v1/kv_connector/unit/offloading_connector/test_worker_metadata.py delete mode 100644 tests/v1/kv_connector/unit/offloading_connector/utils.py delete mode 100644 tests/v1/kv_connector/unit/test_cache_pollution_prevention.py delete mode 100644 tests/v1/kv_connector/unit/test_config.py delete mode 100644 tests/v1/kv_connector/unit/test_error_propagation.py delete mode 100644 tests/v1/kv_connector/unit/test_handshake_pp_aggregation.py delete mode 100644 tests/v1/kv_connector/unit/test_hidden_states_connector.py delete mode 100644 tests/v1/kv_connector/unit/test_invalid_blocks_correctness.py delete mode 100644 tests/v1/kv_connector/unit/test_kv_connector_lifecycle.py delete mode 100644 tests/v1/kv_connector/unit/test_kv_load_failure_recovery.py delete mode 100644 tests/v1/kv_connector/unit/test_mooncake_proxy.py delete mode 100644 tests/v1/kv_connector/unit/test_output_aggregator.py delete mode 100644 tests/v1/kv_connector/unit/test_remote_decode_lifecycle.py delete mode 100644 tests/v1/kv_connector/unit/test_scheduler_kv_connector_override.py delete mode 100644 tests/v1/kv_connector/unit/test_transfer_topology_sharded.py delete mode 100644 tests/v1/kv_connector/unit/utils.py delete mode 100644 tests/v1/kv_offload/test_file_mapper.py delete mode 100644 tests/v1/kv_offload/tiering/__init__.py delete mode 100644 tests/v1/kv_offload/tiering/p2p/__init__.py delete mode 100644 tests/v1/kv_offload/tiering/p2p/conftest.py delete mode 100644 tests/v1/kv_offload/tiering/p2p/p2p_connector_proxy.py delete mode 100755 tests/v1/kv_offload/tiering/p2p/run_accuracy_test.sh delete mode 100644 tests/v1/kv_offload/tiering/p2p/test_data_transport.py delete mode 100644 tests/v1/kv_offload/tiering/p2p/test_manager.py delete mode 100644 tests/v1/kv_offload/tiering/p2p/test_sessions.py delete mode 100644 tests/v1/kv_offload/tiering/p2p/test_zmq_transport.py delete mode 100644 tests/v1/kv_offload/tiering/test_async_lookup.py delete mode 100644 tests/v1/kv_offload/tiering/test_backpressure.py delete mode 100644 tests/v1/kv_offload/tiering/test_factory.py delete mode 100644 tests/v1/kv_offload/tiering/test_fs_tier.py delete mode 100644 tests/v1/kv_offload/tiering/test_kvcr_tier.py delete mode 100644 tests/v1/kv_offload/tiering/test_metrics.py delete mode 100644 tests/v1/kv_offload/tiering/test_obj_tier.py delete mode 100644 tests/v1/kv_offload/tiering/test_tier_poll_thread.py delete mode 100644 tests/v1/kv_offload/tiering/test_tiering_offloading.py delete mode 100644 tests/v1/structured_output/test_backend_xgrammar_lark.py delete mode 100644 vllm/cute_utils/cvt.py delete mode 100644 vllm/cute_utils/mbarrier.py delete mode 100644 vllm/distributed/kv_transfer/kv_connector/v1/example_hidden_states_connector.py delete mode 100644 vllm/entrypoints/anthropic/__init__.py delete mode 100644 vllm/entrypoints/anthropic/api_router.py delete mode 100644 vllm/entrypoints/anthropic/protocol.py delete mode 100644 vllm/entrypoints/anthropic/serving.py delete mode 100644 vllm/entrypoints/cli/launch.py delete mode 100644 vllm/entrypoints/cli/run_batch.py delete mode 100644 vllm/entrypoints/cohere/__init__.py delete mode 100644 vllm/entrypoints/cohere/api_router.py delete mode 100644 vllm/entrypoints/cohere/cohere_chat_message.py delete mode 100644 vllm/entrypoints/cohere/protocol.py delete mode 100644 vllm/entrypoints/cohere/serving.py delete mode 100644 vllm/entrypoints/generate/factories.py delete mode 100644 vllm/entrypoints/generate/structured_decisions/__init__.py delete mode 100644 vllm/entrypoints/generate/structured_decisions/api_router.py delete mode 100644 vllm/entrypoints/generate/structured_decisions/protocol.py delete mode 100644 vllm/entrypoints/generate/structured_decisions/serving.py delete mode 100644 vllm/entrypoints/launchers/render/__init__.py delete mode 100644 vllm/entrypoints/launchers/render/app_state.py delete mode 100644 vllm/entrypoints/launchers/render/entry.py delete mode 100644 vllm/entrypoints/launchers/run_batch.py delete mode 100644 vllm/entrypoints/openai/decisions/__init__.py delete mode 100644 vllm/entrypoints/openai/decisions/adapters.py delete mode 100644 vllm/entrypoints/openai/decisions/api_router.py delete mode 100644 vllm/entrypoints/openai/decisions/protocol.py delete mode 100644 vllm/entrypoints/openai/decisions/question_types.py delete mode 100644 vllm/entrypoints/openai/decisions/serving.py delete mode 100644 vllm/entrypoints/openai/decisions/state.py delete mode 100644 vllm/entrypoints/openai/decisions/strategies.py delete mode 100644 vllm/entrypoints/openai/run_batch.py delete mode 100644 vllm/entrypoints/rl/__init__.py delete mode 100644 vllm/entrypoints/rl/factories.py delete mode 100644 vllm/entrypoints/rl/offline.py delete mode 100644 vllm/entrypoints/rl/online/__init__.py delete mode 100644 vllm/entrypoints/rl/online/api_router.py delete mode 100644 vllm/entrypoints/rl/online/metrics.py delete mode 100644 vllm/entrypoints/scale_out/derender/__init__.py delete mode 100644 vllm/entrypoints/scale_out/derender/api_router.py delete mode 100644 vllm/entrypoints/scale_out/derender/serving.py delete mode 100644 vllm/entrypoints/scale_out/render/__init__.py delete mode 100644 vllm/entrypoints/scale_out/render/api_router.py delete mode 100644 vllm/entrypoints/scale_out/render/serving.py delete mode 100644 vllm/entrypoints/serve/elastic_ep/api_router.py delete mode 100644 vllm/entrypoints/serve/instrumentator/offline_docs.py delete mode 100644 vllm/entrypoints/serve/instrumentator/static/swagger-ui-bundle.js delete mode 100644 vllm/entrypoints/serve/instrumentator/static/swagger-ui.css delete mode 100644 vllm/entrypoints/serve/sagemaker/__init__.py delete mode 100644 vllm/entrypoints/serve/sagemaker/api_router.py delete mode 100644 vllm/entrypoints/speech_to_text/realtime/metrics.py delete mode 100644 vllm/kernels/helion/configs/dynamic_per_token_scaled_fp8_quant/nvidia_b200.json delete mode 100644 vllm/kernels/helion/configs/dynamic_per_token_scaled_fp8_quant/nvidia_h100.json delete mode 100644 vllm/kernels/helion/configs/fused_qk_norm_rope/nvidia_b200.json delete mode 100644 vllm/kernels/helion/configs/fused_qk_norm_rope/nvidia_h100.json delete mode 100644 vllm/kernels/helion/configs/per_token_group_fp8_quant/nvidia_b200.json delete mode 100644 vllm/kernels/helion/configs/per_token_group_fp8_quant/nvidia_h100.json delete mode 100644 vllm/kernels/helion/configs/rms_norm_dynamic_per_token_quant/nvidia_b200.json delete mode 100644 vllm/kernels/helion/configs/rms_norm_dynamic_per_token_quant/nvidia_h100.json delete mode 100644 vllm/kernels/helion/configs/rms_norm_per_block_quant/nvidia_b200.json delete mode 100644 vllm/kernels/helion/configs/rms_norm_per_block_quant/nvidia_h100.json delete mode 100644 vllm/kernels/helion/configs/silu_and_mul_per_block_quant/nvidia_b200.json delete mode 100644 vllm/kernels/helion/configs/silu_and_mul_per_block_quant/nvidia_h100.json delete mode 100644 vllm/kernels/helion/configs/silu_mul_fp8/nvidia_h100.json delete mode 100644 vllm/kernels/helion/configs/silu_mul_fp8/nvidia_h200.json delete mode 100644 vllm/lora/ops/triton_ops/fp8_kernel_utils.py delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=1,N=14336,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=1,N=14336,device_name=NVIDIA_A100-SXM4-80GB.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=1,N=1792,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=1,N=1792,device_name=NVIDIA_A100-SXM4-80GB.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=1,N=1792,device_name=NVIDIA_H100_80GB_HBM3,dtype=int8_w8a16.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=1,N=3072,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=1,N=3072,device_name=NVIDIA_H100_80GB_HBM3,dtype=int8_w8a16.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=1,N=3072,device_name=NVIDIA_H100_80GB_HBM3.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=1,N=3072,device_name=NVIDIA_H200,dtype=int8_w8a16.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=1,N=3584,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=1,N=3584,device_name=NVIDIA_A100-SXM4-80GB.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=1,N=3584,device_name=NVIDIA_H100_80GB_HBM3,dtype=int8_w8a16.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=1,N=7168,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=1,N=7168,device_name=NVIDIA_A100-SXM4-80GB.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=1,N=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=int8_w8a16.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=128,N=1024,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=128,N=1024,device_name=AMD_Instinct_MI300X.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=128,N=1024,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=128,N=1024,device_name=NVIDIA_H100,dtype=fp8_w8a8.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=128,N=1024,device_name=NVIDIA_H200,dtype=fp8_w8a8.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=128,N=1024,device_name=NVIDIA_H200.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=128,N=1856,device_name=NVIDIA_B200.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=128,N=1856,device_name=NVIDIA_H100_80GB_HBM3.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=128,N=1856,device_name=NVIDIA_H200,dtype=fp8_w8a8.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=128,N=1856,device_name=NVIDIA_L40S.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=128,N=192,device_name=NVIDIA_A100-SXM4-80GB.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=128,N=192,device_name=NVIDIA_H100_80GB_HBM3.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=128,N=192,device_name=NVIDIA_H20-3e.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=128,N=192,device_name=NVIDIA_H20.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=128,N=192,device_name=NVIDIA_H200.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=128,N=192,device_name=NVIDIA_H800,dtype=fp8_w8a8.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=128,N=2880,device_name=NVIDIA_H100_80GB_HBM3.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=128,N=352,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=128,N=384,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=128,N=384,device_name=Intel(R)_Arc(TM)_Pro_B70_Graphics.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=128,N=384,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=128,N=384,device_name=NVIDIA_GB200,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=128,N=384,device_name=NVIDIA_H100_80GB_HBM3.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=128,N=384,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=128,N=384,device_name=NVIDIA_H20-3e,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=128,N=384,device_name=NVIDIA_H20-3e.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=128,N=384,device_name=NVIDIA_H20.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=128,N=384,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=128,N=384,device_name=NVIDIA_H200.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=128,N=512,device_name=AMD_Radeon_R9700,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=128,N=512,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=128,N=512,device_name=NVIDIA_B200,dtype=fp8_w8a8.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=128,N=512,device_name=NVIDIA_B200.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=128,N=512,device_name=NVIDIA_GB200,dtype=fp8_w8a8.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=128,N=512,device_name=NVIDIA_H100_80GB_HBM3.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=128,N=512,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=128,N=512,device_name=NVIDIA_H200,dtype=fp8_w8a8.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=128,N=512,device_name=NVIDIA_H200.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=128,N=704,device_name=NVIDIA_B200,dtype=fp8_w8a8.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=128,N=704,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=128,N=704,device_name=NVIDIA_RTX_PRO_6000_Blackwell_Workstation_Edition,dtype=fp8_w8a8.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=128,N=768,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=128,N=768,device_name=AMD_Instinct_MI308X.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=128,N=768,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=128,N=768,device_name=NVIDIA_B200.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=128,N=768,device_name=NVIDIA_GB200,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=128,N=768,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=128,N=768,device_name=NVIDIA_H20-3e,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=128,N=768,device_name=NVIDIA_H20.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=128,N=768,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=128,N=768,device_name=NVIDIA_H200.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=128,N=768,device_name=Radeon_8060S_Graphics,dtype=int4_w4a16.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=128,N=8960,device_name=NVIDIA_H100_80GB_HBM3,dtype=bf16.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=128,N=8960,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=128,N=928,device_name=NVIDIA_H100_80GB_HBM3.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=128,N=928,device_name=NVIDIA_L40S.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=128,N=96,device_name=NVIDIA_H20.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=128,N=96,device_name=NVIDIA_H200,dtype=fp8_w8a8.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=128,N=96,device_name=NVIDIA_H200.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=129,N=704,device_name=NVIDIA_RTX_PRO_6000_Blackwell_Workstation_Edition,dtype=fp8_w8a8.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=16,N=1024,device_name=AMD_Instinct_MI300X.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=16,N=1024,device_name=NVIDIA_B200,dtype=fp8_w8a8.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=16,N=1024,device_name=NVIDIA_B200.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=16,N=1024,device_name=NVIDIA_H100.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=16,N=1024,device_name=NVIDIA_H200,dtype=fp8_w8a8.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=16,N=1024,device_name=NVIDIA_H200.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=16,N=1344,device_name=NVIDIA_A100-SXM4-40GB.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=16,N=1344,device_name=NVIDIA_A100-SXM4-80GB.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=16,N=1344,device_name=NVIDIA_H100_80GB_HBM3.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=16,N=14336,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=16,N=14336,device_name=NVIDIA_A100-SXM4-80GB.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=16,N=14336,device_name=NVIDIA_H100_80GB_HBM3,dtype=int8_w8a16.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=16,N=1792,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=16,N=1792,device_name=NVIDIA_A100-SXM4-80GB.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=16,N=1792,device_name=NVIDIA_H100_80GB_HBM3,dtype=int8_w8a16.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=16,N=1792,device_name=NVIDIA_H100_80GB_HBM3.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=16,N=2048,device_name=NVIDIA_H200,dtype=fp8_w8a8.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=16,N=2048,device_name=NVIDIA_H200.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=16,N=2688,device_name=NVIDIA_A100-SXM4-80GB.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=16,N=2688,device_name=NVIDIA_H100_80GB_HBM3.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=16,N=3072,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=16,N=3072,device_name=NVIDIA_H100_80GB_HBM3,dtype=float8.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=16,N=3072,device_name=NVIDIA_H100_80GB_HBM3,dtype=int8_w8a16.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=16,N=3072,device_name=NVIDIA_H200,dtype=int8_w8a16.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=16,N=3200,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=16,N=3584,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=16,N=3584,device_name=NVIDIA_A100-SXM4-80GB.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=16,N=3584,device_name=NVIDIA_H100_80GB_HBM3,dtype=int8_w8a16.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=16,N=4096,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=16,N=4096,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=16,N=6400,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=16,N=7168,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=16,N=7168,device_name=NVIDIA_A100-SXM4-80GB.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=16,N=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=float8.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=16,N=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=int8_w8a16.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=16,N=800,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=160,N=192,device_name=AMD_Instinct_MI300X.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=160,N=192,device_name=AMD_Instinct_MI350_OAM,dtype=fp8_w8a8.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=160,N=192,device_name=NVIDIA_A800-SXM4-80GB.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=160,N=192,device_name=NVIDIA_B300_SXM6_AC,dtype=fp8_w8a8.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=160,N=192,device_name=NVIDIA_H20-3e.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=160,N=192,device_name=NVIDIA_H200,dtype=fp8_w8a8.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=160,N=320,device_name=NVIDIA_H20-3e.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=160,N=384,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=160,N=384,device_name=AMD_Instinct_MI350_OAM,dtype=fp8_w8a8.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=160,N=384,device_name=AMD_Instinct_MI355_OAM,dtype=fp8_w8a8.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=160,N=384,device_name=NVIDIA_B200,dtype=fp8_w8a8.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=160,N=384,device_name=NVIDIA_B300_SXM6_AC,dtype=fp8_w8a8.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=160,N=640,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=160,N=640,device_name=NVIDIA_GB200,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=160,N=640,device_name=NVIDIA_H100,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=160,N=768,device_name=NVIDIA_B300_SXM6_AC,dtype=fp8_w8a8.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=20,N=1536,device_name=NVIDIA_RTX_PRO_6000_Blackwell_Server_Edition,dtype=fp8_w8a8.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=20,N=2560,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=20,N=2560,device_name=NVIDIA_GB200,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=20,N=2560,device_name=NVIDIA_H100,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=20,N=2560,device_name=NVIDIA_H20-3e,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=256,N=1024,device_name=AMD_Instinct_MI325X,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=256,N=1024,device_name=AMD_Instinct_MI325_OAM,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=256,N=128,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=256,N=128,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=256,N=128,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=256,N=128,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=256,N=128,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=256,N=128,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=256,N=128,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=256,N=256,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=256,N=256,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=256,N=256,device_name=AMD_Instinct_MI325_OAM,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=256,N=256,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=256,N=256,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=256,N=256,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=256,N=256,device_name=NVIDIA_H20-3e,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=256,N=256,device_name=NVIDIA_H20.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=256,N=256,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=256,N=256,device_name=NVIDIA_L20,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=256,N=384,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=256,N=384,device_name=NVIDIA_RTX_PRO_6000_Blackwell_Server_Edition,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=256,N=512,device_name=AMD_Instinct_MI325_OAM,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=256,N=512,device_name=NVIDIA_A100_80GB_PCIe.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=256,N=512,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=256,N=512,device_name=NVIDIA_GB10,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=256,N=512,device_name=NVIDIA_H100_80GB_HBM3.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=256,N=512,device_name=NVIDIA_H20.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=256,N=512,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=256,N=512,device_name=NVIDIA_L40S,dtype=fp8_w8a8.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=256,N=512,device_name=NVIDIA_RTX_PRO_6000_Blackwell_Server_Edition,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=256,N=64,device_name=NVIDIA_A800-SXM4-80GB.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=32,N=1408,device_name=NVIDIA_B200.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=32,N=1792,device_name=NVIDIA_H100_80GB_HBM3.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=32,N=2048,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=32,N=2048,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=384,N=128,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=384,N=128,device_name=NVIDIA_GB200,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=384,N=128,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=384,N=256,device_name=AMD_Instinct_MI350X,dtype=int4_w4a16,backend=flydsl.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=384,N=256,device_name=AMD_Instinct_MI350X,dtype=int4_w4a16.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=384,N=256,device_name=AMD_Instinct_MI350_OAM,dtype=int4_w4a16,backend=flydsl.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=384,N=256,device_name=AMD_Instinct_MI350_OAM,dtype=int4_w4a16.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=384,N=256,device_name=AMD_Instinct_MI355X,dtype=int4_w4a16,backend=flydsl.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=384,N=256,device_name=AMD_Instinct_MI355X,dtype=int4_w4a16.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=384,N=256,device_name=AMD_Instinct_MI355_OAM,dtype=int4_w4a16,backend=flydsl.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=384,N=256,device_name=AMD_Instinct_MI355_OAM,dtype=int4_w4a16.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=384,N=256,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=384,N=256,device_name=NVIDIA_GB200,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=384,N=512,device_name=AMD_Instinct_MI350X,dtype=int4_w4a16,backend=flydsl.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=384,N=512,device_name=AMD_Instinct_MI350_OAM,dtype=int4_w4a16,backend=flydsl.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=384,N=512,device_name=AMD_Instinct_MI355X,dtype=int4_w4a16,backend=flydsl.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=384,N=512,device_name=AMD_Instinct_MI355_OAM,dtype=int4_w4a16,backend=flydsl.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=40,N=1536,device_name=NVIDIA_B200,dtype=fp8_w8a8.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=40,N=2560,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=40,N=2560,device_name=NVIDIA_GB200,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=40,N=2560,device_name=NVIDIA_H100,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=512,N=128,device_name=NVIDIA_A100-SXM4-80GB.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=512,N=128,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=512,N=128,device_name=NVIDIA_B200.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=512,N=128,device_name=NVIDIA_GB200,dtype=fp8_w8a8.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=512,N=128,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=512,N=128,device_name=NVIDIA_H100_80GB_HBM3.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=512,N=128,device_name=NVIDIA_H20-3e.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=512,N=128,device_name=NVIDIA_H200.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=512,N=1344,device_name=NVIDIA_B200.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=512,N=160,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[32,32].json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=512,N=192,device_name=NVIDIA_H20-3e.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=512,N=256,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=512,N=256,device_name=NVIDIA_B200.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=512,N=256,device_name=NVIDIA_GB200,dtype=fp8_w8a8.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=512,N=256,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=512,N=256,device_name=NVIDIA_H100_80GB_HBM3.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=512,N=256,device_name=NVIDIA_H20-3e.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=512,N=256,device_name=NVIDIA_H200.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=512,N=320,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[64,64].json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=512,N=512,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=512,N=512,device_name=NVIDIA_B200.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=512,N=512,device_name=NVIDIA_GB10,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=512,N=512,device_name=NVIDIA_GB200,dtype=fp8_w8a8.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=512,N=512,device_name=NVIDIA_H100_80GB_HBM3.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=512,N=512,device_name=NVIDIA_H20-3e.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=512,N=512,device_name=NVIDIA_H200.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=512,N=64,device_name=NVIDIA_A100-SXM4-80GB.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=512,N=64,device_name=NVIDIA_B200.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=512,N=64,device_name=NVIDIA_H20-3e.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=512,N=64,device_name=NVIDIA_H200.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=512,N=672,device_name=NVIDIA_B200.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=60,N=1408,device_name=AMD_Instinct_MI300X.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=60,N=176,device_name=AMD_Instinct_MI300X.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=60,N=352,device_name=AMD_Instinct_MI300X.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=60,N=704,device_name=AMD_Instinct_MI300X.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=62,N=128,device_name=AMD_Instinct_MI300X.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=62,N=256,device_name=AMD_Instinct_MI300X.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=62,N=256,device_name=NVIDIA_H100_80GB_HBM3.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=62,N=512,device_name=AMD_Instinct_MI300X.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=62,N=512,device_name=NVIDIA_H100_80GB_HBM3.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=64,N=1024,device_name=Intel(R)_Arc(TM)_Pro_B70_Graphics.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=64,N=1280,device_name=NVIDIA_A100-SXM4-80GB.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=64,N=1280,device_name=NVIDIA_A800-SXM4-80GB.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=64,N=1280,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=64,N=1280,device_name=NVIDIA_H100_80GB_HBM3.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=64,N=1280,device_name=NVIDIA_H200,dtype=fp8_w8a8.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=64,N=1280,device_name=NVIDIA_H200.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=64,N=1408,device_name=NVIDIA_B200.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=64,N=1536,device_name=NVIDIA_H100_80GB_HBM3.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=64,N=1536,device_name=NVIDIA_H20,dtype=fp8_w8a8.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=64,N=1536,device_name=NVIDIA_RTX_PRO_6000_Blackwell_Server_Edition,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=64,N=2560,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=64,N=2560,device_name=NVIDIA_H200,dtype=fp8_w8a8.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=64,N=2560,device_name=NVIDIA_H200.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=64,N=3072,device_name=NVIDIA_H20,dtype=fp8_w8a8.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=64,N=3072,device_name=NVIDIA_H20.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=64,N=320,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=64,N=320,device_name=NVIDIA_H100_80GB_HBM3.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=64,N=320,device_name=NVIDIA_H200,dtype=fp8_w8a8.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=64,N=320,device_name=NVIDIA_H200.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=64,N=384,device_name=NVIDIA_H20,dtype=fp8_w8a8.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=64,N=384,device_name=NVIDIA_H20.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=64,N=512,device_name=Intel(R)_Arc(TM)_Pro_B70_Graphics.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=64,N=512,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=64,N=640,device_name=NVIDIA_A100-SXM4-80GB.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=64,N=640,device_name=NVIDIA_A800-SXM4-80GB.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=64,N=640,device_name=NVIDIA_GeForce_RTX_4090,dtype=fp8_w8a8.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=64,N=640,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=64,N=640,device_name=NVIDIA_H100_80GB_HBM3.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=64,N=640,device_name=NVIDIA_H200,dtype=fp8_w8a8.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=64,N=640,device_name=NVIDIA_H200.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=64,N=768,device_name=AMD_Radeon_R9700,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=64,N=768,device_name=NVIDIA_H100_PCIe,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=64,N=768,device_name=NVIDIA_H20,dtype=fp8_w8a8.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=64,N=768,device_name=NVIDIA_H20.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=64,N=896,device_name=NVIDIA_H20.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=64,N=8960,device_name=NVIDIA_H100_80GB_HBM3,dtype=bf16.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=64,N=8960,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=72,N=192,device_name=AMD_Instinct_MI300X.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=72,N=384,device_name=AMD_Instinct_MI300X.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=72,N=384,device_name=NVIDIA_H100_80GB_HBM3.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=72,N=768,device_name=AMD_Instinct_MI300X.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=72,N=768,device_name=NVIDIA_H100_80GB_HBM3.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=8,N=14336,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=8,N=14336,device_name=AMD_Instinct_MI300X.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=8,N=14336,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=8,N=14336,device_name=AMD_Instinct_MI325X.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=8,N=14336,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=8,N=14336,device_name=NVIDIA_H200,dtype=fp8_w8a8.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=8,N=14336,device_name=NVIDIA_H200.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=8,N=16384,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=8,N=16384,device_name=AMD_Instinct_MI300X.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=8,N=16384,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=8,N=16384,device_name=AMD_Instinct_MI325X.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=8,N=1792,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=8,N=1792,device_name=AMD_Instinct_MI300X.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=8,N=1792,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=8,N=1792,device_name=AMD_Instinct_MI325X.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=8,N=1792,device_name=NVIDIA_A100-SXM4-40GB.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=8,N=1792,device_name=NVIDIA_A100-SXM4-80GB.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=8,N=1792,device_name=NVIDIA_H100_80GB_HBM3.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=8,N=1792,device_name=NVIDIA_H200,dtype=fp8_w8a8.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=8,N=1792,device_name=NVIDIA_H200.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=8,N=2048,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=8,N=2048,device_name=AMD_Instinct_MI300X.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=8,N=2048,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=8,N=2048,device_name=AMD_Instinct_MI325X.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=8,N=2048,device_name=NVIDIA_A100-SXM4-80GB.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=8,N=2048,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=8,N=2048,device_name=NVIDIA_H100_80GB_HBM3.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=8,N=2048,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=8,N=2048,device_name=NVIDIA_H200,dtype=fp8_w8a8.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=8,N=2048,device_name=NVIDIA_H200.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=8,N=3584,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=8,N=3584,device_name=AMD_Instinct_MI300X.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=8,N=3584,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=8,N=3584,device_name=AMD_Instinct_MI325X.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=8,N=3584,device_name=NVIDIA_A100-SXM4-40GB.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=8,N=3584,device_name=NVIDIA_A100-SXM4-80GB.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=8,N=3584,device_name=NVIDIA_GeForce_RTX_4090,dtype=fp8_w8a8.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=8,N=3584,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=8,N=3584,device_name=NVIDIA_H100_80GB_HBM3.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=8,N=3584,device_name=NVIDIA_H200,dtype=fp8_w8a8.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=8,N=3584,device_name=NVIDIA_H200.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=8,N=3584,device_name=NVIDIA_L40S.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=8,N=4096,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=8,N=4096,device_name=AMD_Instinct_MI300X.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=8,N=4096,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=8,N=4096,device_name=AMD_Instinct_MI325X.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=8,N=4096,device_name=NVIDIA_A100-SXM4-80GB.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=8,N=4096,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=8,N=4096,device_name=NVIDIA_H100_80GB_HBM3.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=8,N=4096,device_name=NVIDIA_H200,dtype=fp8_w8a8.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=8,N=4096,device_name=NVIDIA_H200.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=8,N=7168,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=8,N=7168,device_name=AMD_Instinct_MI300X.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=8,N=7168,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=8,N=7168,device_name=AMD_Instinct_MI325X.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=8,N=7168,device_name=NVIDIA_A100-SXM4-80GB.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=8,N=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=8,N=7168,device_name=NVIDIA_H100_80GB_HBM3.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=8,N=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=8,N=7168,device_name=NVIDIA_H200.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=8,N=8192,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=8,N=8192,device_name=AMD_Instinct_MI300X.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=8,N=8192,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=8,N=8192,device_name=AMD_Instinct_MI325X.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=8,N=8192,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/E=8,N=8192,device_name=NVIDIA_H200,dtype=fp8_w8a8.json delete mode 100644 vllm/model_executor/layers/fused_moe/configs/README delete mode 100644 vllm/model_executor/layers/mamba/ops/configs/selective_state_update/headdim=128,dstate=256,device_name=Intel(R)_Arc(TM)_Pro_B60_Graphics,cache_dtype=float16.json delete mode 100644 vllm/model_executor/layers/mamba/ops/configs/selective_state_update/headdim=128,dstate=256,device_name=Intel(R)_Arc(TM)_Pro_B60_Graphics,cache_dtype=float32.json delete mode 100644 vllm/model_executor/layers/mamba/ops/configs/selective_state_update/headdim=128,dstate=256,device_name=Intel(R)_Arc(TM)_Pro_B70_Graphics,cache_dtype=float16.json delete mode 100644 vllm/model_executor/layers/mamba/ops/configs/selective_state_update/headdim=128,dstate=256,device_name=Intel(R)_Arc(TM)_Pro_B70_Graphics,cache_dtype=float32.json delete mode 100644 vllm/model_executor/layers/mamba/ops/configs/selective_state_update/headdim=64,dstate=128,device_name=AMD_Instinct_MI300X,cache_dtype=float16.json delete mode 100644 vllm/model_executor/layers/mamba/ops/configs/selective_state_update/headdim=64,dstate=128,device_name=AMD_Instinct_MI300X,cache_dtype=float32.json delete mode 100644 vllm/model_executor/layers/mamba/ops/configs/selective_state_update/headdim=64,dstate=128,device_name=AMD_Instinct_MI325X,cache_dtype=float16.json delete mode 100644 vllm/model_executor/layers/mamba/ops/configs/selective_state_update/headdim=64,dstate=128,device_name=AMD_Instinct_MI325X,cache_dtype=float32.json delete mode 100644 vllm/model_executor/layers/mamba/ops/configs/selective_state_update/headdim=64,dstate=128,device_name=AMD_Instinct_MI350_OAM,cache_dtype=float16.json delete mode 100644 vllm/model_executor/layers/mamba/ops/configs/selective_state_update/headdim=64,dstate=128,device_name=AMD_Instinct_MI350_OAM,cache_dtype=float32.json delete mode 100644 vllm/model_executor/layers/mamba/ops/configs/selective_state_update/headdim=64,dstate=128,device_name=AMD_Instinct_MI355_OAM,cache_dtype=float16.json delete mode 100644 vllm/model_executor/layers/mamba/ops/configs/selective_state_update/headdim=64,dstate=128,device_name=AMD_Instinct_MI355_OAM,cache_dtype=float32.json delete mode 100644 vllm/model_executor/layers/mamba/ops/configs/selective_state_update/headdim=64,dstate=128,device_name=Intel(R)_Arc(TM)_Pro_B60_Graphics,cache_dtype=float16.json delete mode 100644 vllm/model_executor/layers/mamba/ops/configs/selective_state_update/headdim=64,dstate=128,device_name=Intel(R)_Arc(TM)_Pro_B60_Graphics,cache_dtype=float32.json delete mode 100644 vllm/model_executor/layers/mamba/ops/configs/selective_state_update/headdim=64,dstate=128,device_name=Intel(R)_Arc(TM)_Pro_B70_Graphics,cache_dtype=float16.json delete mode 100644 vllm/model_executor/layers/mamba/ops/configs/selective_state_update/headdim=64,dstate=128,device_name=Intel(R)_Arc(TM)_Pro_B70_Graphics,cache_dtype=float32.json delete mode 100644 vllm/model_executor/layers/mamba/ops/configs/selective_state_update/headdim=64,dstate=128,device_name=NVIDIA_B200,cache_dtype=float16.json delete mode 100644 vllm/model_executor/layers/mamba/ops/configs/selective_state_update/headdim=64,dstate=128,device_name=NVIDIA_B200,cache_dtype=float32.json delete mode 100644 vllm/model_executor/layers/mamba/ops/configs/selective_state_update/headdim=64,dstate=128,device_name=NVIDIA_GB200,cache_dtype=float16.json delete mode 100644 vllm/model_executor/layers/mamba/ops/configs/selective_state_update/headdim=64,dstate=128,device_name=NVIDIA_GB200,cache_dtype=float32.json delete mode 100644 vllm/model_executor/layers/mamba/ops/configs/selective_state_update/headdim=64,dstate=128,device_name=NVIDIA_H100_80GB_HBM3,cache_dtype=float16.json delete mode 100644 vllm/model_executor/layers/mamba/ops/configs/selective_state_update/headdim=64,dstate=128,device_name=NVIDIA_H100_80GB_HBM3,cache_dtype=float32.json delete mode 100644 vllm/model_executor/layers/mamba/ops/configs/selective_state_update/headdim=64,dstate=128,device_name=NVIDIA_H200,cache_dtype=float16.json delete mode 100644 vllm/model_executor/layers/mamba/ops/configs/selective_state_update/headdim=64,dstate=128,device_name=NVIDIA_H200,cache_dtype=float32.json delete mode 100644 vllm/model_executor/layers/mamba/ops/configs/selective_state_update/headdim=64,dstate=128,device_name=NVIDIA_RTX_PRO_6000_Blackwell_Server_Edition,cache_dtype=float16.json delete mode 100644 vllm/model_executor/layers/mamba/ops/configs/selective_state_update/headdim=64,dstate=128,device_name=NVIDIA_RTX_PRO_6000_Blackwell_Server_Edition,cache_dtype=float32.json delete mode 100644 vllm/model_executor/layers/mamba/ops/configs/selective_state_update/headdim=64,dstate=256,device_name=Intel(R)_Arc(TM)_Pro_B60_Graphics,cache_dtype=float16.json delete mode 100644 vllm/model_executor/layers/mamba/ops/configs/selective_state_update/headdim=64,dstate=256,device_name=Intel(R)_Arc(TM)_Pro_B60_Graphics,cache_dtype=float32.json delete mode 100644 vllm/model_executor/layers/mamba/ops/configs/selective_state_update/headdim=64,dstate=256,device_name=Intel(R)_Arc(TM)_Pro_B70_Graphics,cache_dtype=float16.json delete mode 100644 vllm/model_executor/layers/mamba/ops/configs/selective_state_update/headdim=64,dstate=256,device_name=Intel(R)_Arc(TM)_Pro_B70_Graphics,cache_dtype=float32.json delete mode 100644 vllm/model_executor/layers/mamba/ops/configs/selective_state_update/headdim=64,dstate=64,device_name=Intel(R)_Arc(TM)_Pro_B60_Graphics,cache_dtype=float16.json delete mode 100644 vllm/model_executor/layers/mamba/ops/configs/selective_state_update/headdim=64,dstate=64,device_name=Intel(R)_Arc(TM)_Pro_B60_Graphics,cache_dtype=float32.json delete mode 100644 vllm/model_executor/layers/mamba/ops/configs/selective_state_update/headdim=64,dstate=64,device_name=Intel(R)_Arc(TM)_Pro_B70_Graphics,cache_dtype=float16.json delete mode 100644 vllm/model_executor/layers/mamba/ops/configs/selective_state_update/headdim=64,dstate=64,device_name=Intel(R)_Arc(TM)_Pro_B70_Graphics,cache_dtype=float32.json delete mode 100644 vllm/model_executor/layers/quantization/fbgemm_fp8.py delete mode 100644 vllm/model_executor/layers/quantization/fp_quant.py delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=1024,K=2048,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=10240,K=5120,device_name=NVIDIA_L40S,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=12288,K=2048,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=12288,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=12288,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=1536,K=1536,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=1536,K=1536,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=1536,K=1536,device_name=AMD_Instinct_MI325_OAM,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=1536,K=1536,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=1536,K=1536,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=1536,K=1536,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=1536,K=1536,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=1536,K=1536,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=1536,K=7168,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=1536,K=7168,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=1536,K=7168,device_name=AMD_Instinct_MI325_OAM,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=1536,K=7168,device_name=Intel(R)_Arc(TM)_Pro_B70_Graphics,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=1536,K=7168,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=1536,K=7168,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=1536,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=1536,K=7168,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=1536,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=1536,K=7168,device_name=NVIDIA_L20,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=1536,K=7168,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=2048,K=11008,device_name=Intel(R)_Arc(TM)_Pro_B70_Graphics,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=2048,K=2048,device_name=Intel(R)_Arc(TM)_Pro_B70_Graphics,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=2048,K=4096,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=2048,K=512,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=2048,K=512,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=2048,K=512,device_name=AMD_Instinct_MI325_OAM,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=2048,K=512,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=2048,K=512,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=2048,K=512,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=2048,K=512,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=2048,K=512,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=2048,K=512,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=2112,K=7168,device_name=Intel(R)_Arc(TM)_Pro_B70_Graphics,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=2112,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=2112,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=22016,K=2048,device_name=Intel(R)_Arc(TM)_Pro_B70_Graphics,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=2304,K=7168,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=2304,K=7168,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=2304,K=7168,device_name=AMD_Instinct_MI325_OAM,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=2304,K=7168,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=2304,K=7168,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=2304,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=2304,K=7168,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=2304,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=2304,K=7168,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=24576,K=1536,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=24576,K=1536,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=24576,K=4096,device_name=Intel(R)_Arc(TM)_Pro_B70_Graphics,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=24576,K=7168,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=24576,K=7168,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=24576,K=7168,device_name=AMD_Instinct_MI325_OAM,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=24576,K=7168,device_name=Intel(R)_Arc(TM)_Pro_B70_Graphics,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=24576,K=7168,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=24576,K=7168,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=24576,K=7168,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=24576,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=24576,K=7168,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=24576,K=7168,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=24576,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=24576,K=7168,device_name=NVIDIA_L20,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=24576,K=7168,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=256,K=7168,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=256,K=7168,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=256,K=7168,device_name=AMD_Instinct_MI325_OAM,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=256,K=7168,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=256,K=7168,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=256,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=256,K=7168,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=256,K=7168,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=28672,K=4096,device_name=Intel(R)_Arc(TM)_Pro_B70_Graphics,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=3072,K=1536,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=3072,K=1536,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=3072,K=1536,device_name=AMD_Instinct_MI325_OAM,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=3072,K=1536,device_name=Intel(R)_Arc(TM)_Pro_B70_Graphics,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=3072,K=1536,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=3072,K=1536,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=3072,K=1536,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=3072,K=1536,device_name=NVIDIA_L20,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=3072,K=2048,device_name=Intel(R)_Arc(TM)_Pro_B70_Graphics,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=3072,K=7168,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=3072,K=7168,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=3072,K=7168,device_name=AMD_Instinct_MI325_OAM,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=3072,K=7168,device_name=Intel(R)_Arc(TM)_Pro_B70_Graphics,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=3072,K=7168,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=3072,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=3072,K=7168,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=3072,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=3072,K=7168,device_name=NVIDIA_L20,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=32768,K=512,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=32768,K=512,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=32768,K=512,device_name=AMD_Instinct_MI325_OAM,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=32768,K=512,device_name=Intel(R)_Arc(TM)_Pro_B70_Graphics,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=32768,K=512,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=32768,K=512,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=32768,K=512,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=32768,K=512,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=32768,K=512,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=32768,K=512,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=32768,K=512,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=32768,K=512,device_name=NVIDIA_L20,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=32768,K=512,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=36864,K=7168,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=36864,K=7168,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=36864,K=7168,device_name=AMD_Instinct_MI325_OAM,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=36864,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=36864,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=4096,K=12288,device_name=Intel(R)_Arc(TM)_Pro_B70_Graphics,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=4096,K=14336,device_name=Intel(R)_Arc(TM)_Pro_B70_Graphics,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=4096,K=4096,device_name=Intel(R)_Arc(TM)_Pro_B70_Graphics,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=4096,K=512,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=4096,K=512,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=4096,K=512,device_name=AMD_Instinct_MI325_OAM,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=4096,K=512,device_name=Intel(R)_Arc(TM)_Pro_B70_Graphics,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=4096,K=512,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=4096,K=512,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=4096,K=512,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=4096,K=512,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=4096,K=512,device_name=NVIDIA_L20,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=4096,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=4096,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=4608,K=7168,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=4608,K=7168,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=4608,K=7168,device_name=AMD_Instinct_MI325_OAM,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=4608,K=7168,device_name=Intel(R)_Arc(TM)_Pro_B70_Graphics,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=4608,K=7168,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=4608,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=4608,K=7168,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=4608,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=4608,K=7168,device_name=NVIDIA_L20,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=512,K=7168,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=512,K=7168,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=512,K=7168,device_name=AMD_Instinct_MI325_OAM,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=512,K=7168,device_name=Intel(R)_Arc(TM)_Pro_B70_Graphics,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=512,K=7168,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=512,K=7168,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=512,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=512,K=7168,device_name=NVIDIA_L20,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=5120,K=25600,device_name=NVIDIA_L40S,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=5120,K=8192,device_name=NVIDIA_L40S,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=51200,K=5120,device_name=NVIDIA_L40S,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=576,K=2048,device_name=Intel(R)_Arc(TM)_Pro_B70_Graphics,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=576,K=7168,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=576,K=7168,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=576,K=7168,device_name=AMD_Instinct_MI325_OAM,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=576,K=7168,device_name=Intel(R)_Arc(TM)_Pro_B70_Graphics,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=576,K=7168,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=576,K=7168,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=576,K=7168,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=576,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=576,K=7168,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=576,K=7168,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=576,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=576,K=7168,device_name=NVIDIA_L20,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=576,K=7168,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=6144,K=4096,device_name=Intel(R)_Arc(TM)_Pro_B70_Graphics,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=7168,K=1024,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=7168,K=1024,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=7168,K=1024,device_name=AMD_Instinct_MI325_OAM,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=7168,K=1024,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=7168,K=1024,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=7168,K=1024,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=7168,K=1024,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=7168,K=1024,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=7168,K=1024,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=7168,K=1152,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=7168,K=1152,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=7168,K=1152,device_name=AMD_Instinct_MI325_OAM,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=7168,K=1152,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=7168,K=1152,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=7168,K=1152,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=7168,K=1152,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=7168,K=1152,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=7168,K=1152,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=7168,K=128,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=7168,K=128,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=7168,K=128,device_name=AMD_Instinct_MI325_OAM,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=7168,K=128,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=7168,K=128,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=7168,K=128,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=7168,K=128,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=7168,K=128,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=7168,K=16384,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=7168,K=16384,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=7168,K=16384,device_name=AMD_Instinct_MI325_OAM,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=7168,K=16384,device_name=Intel(R)_Arc(TM)_Pro_B70_Graphics,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=7168,K=16384,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=7168,K=16384,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=7168,K=16384,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=7168,K=16384,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=7168,K=16384,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=7168,K=16384,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=7168,K=16384,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=7168,K=16384,device_name=NVIDIA_L20,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=7168,K=16384,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=7168,K=18432,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=7168,K=18432,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=7168,K=18432,device_name=AMD_Instinct_MI325_OAM,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=7168,K=18432,device_name=Intel(R)_Arc(TM)_Pro_B70_Graphics,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=7168,K=18432,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=7168,K=18432,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=7168,K=18432,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=7168,K=18432,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=7168,K=18432,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=7168,K=18432,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=7168,K=18432,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=7168,K=18432,device_name=NVIDIA_L20,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=7168,K=18432,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=7168,K=2048,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=7168,K=2048,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=7168,K=2048,device_name=AMD_Instinct_MI325_OAM,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=7168,K=2048,device_name=Intel(R)_Arc(TM)_Pro_B70_Graphics,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=7168,K=2048,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=7168,K=2048,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=7168,K=2048,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=7168,K=2048,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=7168,K=2048,device_name=NVIDIA_L20,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=7168,K=2304,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=7168,K=2304,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=7168,K=2304,device_name=AMD_Instinct_MI325_OAM,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=7168,K=2304,device_name=Intel(R)_Arc(TM)_Pro_B70_Graphics,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=7168,K=2304,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=7168,K=2304,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=7168,K=2304,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=7168,K=2304,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=7168,K=2304,device_name=NVIDIA_L20,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=7168,K=256,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=7168,K=256,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=7168,K=256,device_name=AMD_Instinct_MI325_OAM,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=7168,K=256,device_name=Intel(R)_Arc(TM)_Pro_B70_Graphics,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=7168,K=256,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=7168,K=256,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=7168,K=256,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=7168,K=256,device_name=NVIDIA_L20,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=7168,K=8192,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=7168,K=8192,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=7168,K=8192,device_name=AMD_Instinct_MI325_OAM,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=8192,K=1536,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=8192,K=1536,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=8192,K=1536,device_name=AMD_Instinct_MI325_OAM,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/N=9216,K=2048,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128,128].json delete mode 100644 vllm/model_executor/layers/quantization/utils/configs/README.md delete mode 100644 vllm/model_executor/layers/rotary_embedding/bailing_mrope.py delete mode 100644 vllm/model_executor/layers/rotary_embedding/dynamic_ntk_alpha_rope.py delete mode 100644 vllm/model_executor/layers/rotary_embedding/dynamic_ntk_scaling_rope.py delete mode 100644 vllm/model_executor/layers/rotary_embedding/fope.py delete mode 100644 vllm/model_executor/layers/rotary_embedding/gemma4_rope.py delete mode 100644 vllm/model_executor/layers/rotary_embedding/linear_scaling_rope.py delete mode 100644 vllm/model_executor/layers/rotary_embedding/llama3_rope.py delete mode 100644 vllm/model_executor/layers/rotary_embedding/llama4_vision_rope.py delete mode 100644 vllm/model_executor/layers/rotary_embedding/ntk_scaling_rope.py delete mode 100644 vllm/model_executor/layers/rotary_embedding/phi3_long_rope_scaled_rope.py delete mode 100644 vllm/parser/mistral.py delete mode 100644 vllm/plugins/io_processors/__init__.py delete mode 100644 vllm/plugins/io_processors/interface.py delete mode 100644 vllm/renderers/cohere.py delete mode 100644 vllm/renderers/mistral.py delete mode 100644 vllm/renderers/online_derenderer.py delete mode 100644 vllm/tokenizers/mistral.py delete mode 100644 vllm/transformers_utils/chat_templates/template_basic.jinja delete mode 100644 vllm/transformers_utils/chat_templates/template_blip2.jinja delete mode 100644 vllm/transformers_utils/chat_templates/template_chatml.jinja delete mode 100644 vllm/transformers_utils/chat_templates/template_deepseek_ocr.jinja delete mode 100644 vllm/transformers_utils/chat_templates/template_deepseek_vl2.jinja delete mode 100644 vllm/transformers_utils/chat_templates/template_kimi_audio.jinja delete mode 100644 vllm/transformers_utils/chat_templates/template_minicpmv45.jinja delete mode 100644 vllm/transformers_utils/configs/mimo_v2_omni.py delete mode 100644 vllm/transformers_utils/processors/__init__.py delete mode 100644 vllm/triton_utils/dispatcher.py delete mode 100644 vllm/utils/mistral.py delete mode 100644 vllm/v1/attention/ops/paged_attn.py delete mode 100644 vllm/v1/kv_offload/file_mapper.py delete mode 100644 vllm/v1/kv_offload/tiering/__init__.py delete mode 100644 vllm/v1/kv_offload/tiering/async_lookup.py delete mode 100644 vllm/v1/kv_offload/tiering/backpressure.py delete mode 100644 vllm/v1/kv_offload/tiering/base.py delete mode 100644 vllm/v1/kv_offload/tiering/example/__init__.py delete mode 100644 vllm/v1/kv_offload/tiering/example/manager.py delete mode 100644 vllm/v1/kv_offload/tiering/factory.py delete mode 100644 vllm/v1/kv_offload/tiering/fs/__init__.py delete mode 100644 vllm/v1/kv_offload/tiering/fs/io.py delete mode 100644 vllm/v1/kv_offload/tiering/fs/manager.py delete mode 100644 vllm/v1/kv_offload/tiering/fs/thread_pool.py delete mode 100644 vllm/v1/kv_offload/tiering/kvcr/__init__.py delete mode 100644 vllm/v1/kv_offload/tiering/kvcr/manager.py delete mode 100644 vllm/v1/kv_offload/tiering/manager.py delete mode 100644 vllm/v1/kv_offload/tiering/metrics.py delete mode 100644 vllm/v1/kv_offload/tiering/obj/__init__.py delete mode 100644 vllm/v1/kv_offload/tiering/obj/config.py delete mode 100644 vllm/v1/kv_offload/tiering/obj/manager.py delete mode 100644 vllm/v1/kv_offload/tiering/p2p/__init__.py delete mode 100644 vllm/v1/kv_offload/tiering/p2p/control/__init__.py delete mode 100644 vllm/v1/kv_offload/tiering/p2p/control/base.py delete mode 100644 vllm/v1/kv_offload/tiering/p2p/control/zmq.py delete mode 100644 vllm/v1/kv_offload/tiering/p2p/data/__init__.py delete mode 100644 vllm/v1/kv_offload/tiering/p2p/data/base.py delete mode 100644 vllm/v1/kv_offload/tiering/p2p/data/nixl.py delete mode 100644 vllm/v1/kv_offload/tiering/p2p/manager.py delete mode 100644 vllm/v1/kv_offload/tiering/p2p/session/__init__.py delete mode 100644 vllm/v1/kv_offload/tiering/p2p/session/client.py delete mode 100644 vllm/v1/kv_offload/tiering/p2p/session/protocol.py delete mode 100644 vllm/v1/kv_offload/tiering/p2p/session/server.py delete mode 100644 vllm/v1/kv_offload/tiering/p2p/session/session.py delete mode 100644 vllm/v1/kv_offload/tiering/spec.py diff --git a/.agents/skills/transformers-upgrade/SKILL.md b/.agents/skills/transformers-upgrade/SKILL.md index 847b292cc5..959d5281f2 100644 --- a/.agents/skills/transformers-upgrade/SKILL.md +++ b/.agents/skills/transformers-upgrade/SKILL.md @@ -183,9 +183,8 @@ Add `tests/models/test_initialization.py -k ""` where a GPU is available, and a model eval for any model whose resolved config changed. Models in `vllm/models/` are high priority: whenever their config or model -code changes, run GSM8k (`tests/evals/gsm8k/`) and GPQA Diamond -(`tests/evals/gpt_oss/`) on `main` and the branch, even when the dumps show -no diff, and put both scores in the PR. FlashInfer autotuning picks kernel +code changes, run GSM8k (`tests/evals/gsm8k/`) on `main` and the branch, +even when the dumps show no diff, and put the score in the PR. FlashInfer tactics afresh on every server start, which moved DeepSeek-V4-Flash GSM8k by about 1 point; pass `--kernel-config '{"enable_flashinfer_autotune": false}'` to both servers so greedy outputs are comparable. diff --git a/CMakeLists.txt b/CMakeLists.txt index c84e2b9eed..e756127a6f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -168,11 +168,6 @@ define_extension_target( # if (NOT VLLM_TARGET_DEVICE STREQUAL "cuda" AND NOT VLLM_TARGET_DEVICE STREQUAL "rocm") - if (VLLM_TARGET_DEVICE STREQUAL "cpu") - include(${CMAKE_CURRENT_LIST_DIR}/cmake/cpu_extension.cmake) - else() - return() - endif() return() endif() @@ -415,7 +410,6 @@ if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP") "csrc/cuda_view.cu" "csrc/cuda_utils_kernels.cu" "csrc/activation_kernels.cu" - "csrc/muse_spark_mul_tanh_soft_cap_kernel.cu" "csrc/ngram_embedding_kernels.cu" "csrc/vocab_parallel_embedding_kernels.cu" "csrc/quantization/activation_kernels.cu" @@ -424,9 +418,7 @@ if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP") "csrc/quantization/w8a8/fp8/per_token_group_quant.cu" "csrc/quantization/gptq/q_gemm.cu" "csrc/pos_encoding_kernels.cu" - "csrc/fused_muse_spark_qknorm_rope_kv_insert_kernel.cu" "csrc/fused_qknorm_rope_kernel.cu" - "csrc/fused_minimax_m3_qknorm_rope_kv_insert_kernel.cu" "csrc/layernorm_kernels.cu" "csrc/layernorm_quant_kernels.cu" "csrc/quantization/fused_kernels/fused_layernorm_dynamic_per_token_quant.cu" @@ -439,14 +431,7 @@ if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP") "csrc/cache_kernels_fused.cu" "csrc/custom_all_gather_reduce_scatter.cu" "csrc/custom_all_gather_reduce_scatter_ops.cpp" - "csrc/custom_all_reduce.cu" - "csrc/fused_deepseek_v4_qnorm_rope_kv_insert_kernel.cu" - "csrc/fused_kimi_k3_mla_key_concat_kv_cache_kernel.cu") - - if(VLLM_GPU_LANG STREQUAL "CUDA") - list(APPEND VLLM_STABLE_EXT_SRC - "csrc/fused_qknorm_rope_cache_kernel.cu") - endif() + "csrc/custom_all_reduce.cu") if(VLLM_GPU_LANG STREQUAL "CUDA" AND DEFINED CMAKE_CUDA_COMPILER_VERSION AND @@ -509,8 +494,7 @@ if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP") "csrc/quantization/w8a8/cutlass/scaled_mm_entry.cu" "csrc/quantization/fp4/nvfp4_quant_entry.cu" "csrc/quantization/fp4/nvfp4_scaled_mm_entry.cu" - "csrc/quantization/awq/gemm_kernels.cu" - "csrc/minimax_reduce_rms_kernel.cu") + "csrc/quantization/awq/gemm_kernels.cu") # # Machete kernels @@ -581,30 +565,9 @@ if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP") endif() endif() - # The fused MiniMax-M3 KV insert quantizes NVFP4 with SM100-family-only - # cvt instructions, so its SM10x/11x code is built for the family targets. - set(MINIMAX_M3_FUSED_SRC - "csrc/fused_minimax_m3_qknorm_rope_kv_insert_kernel.cu") - list(REMOVE_ITEM VLLM_STABLE_EXT_SRC "${MINIMAX_M3_FUSED_SRC}") set_gencode_flags_for_srcs( SRCS "${VLLM_STABLE_EXT_SRC}" CUDA_ARCHS "${CUDA_ARCHS}") - if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 13.0) - cuda_archs_loose_intersection(MINIMAX_M3_FUSED_ARCHS - "10.0f;10.7f;11.0f" "${CUDA_ARCHS}") - else() - cuda_archs_loose_intersection(MINIMAX_M3_FUSED_ARCHS - "10.0a;10.1a;10.3a" "${CUDA_ARCHS}") - endif() - foreach(_ARCH ${CUDA_ARCHS}) - if(_ARCH VERSION_LESS 10.0 OR _ARCH VERSION_GREATER_EQUAL 12.0) - list(APPEND MINIMAX_M3_FUSED_ARCHS "${_ARCH}") - endif() - endforeach() - set_gencode_flags_for_srcs( - SRCS "${MINIMAX_M3_FUSED_SRC}" - CUDA_ARCHS "${MINIMAX_M3_FUSED_ARCHS}") - list(APPEND VLLM_STABLE_EXT_SRC "${MINIMAX_M3_FUSED_SRC}") if(COOPERATIVE_TOPK_ARCHS) list(APPEND VLLM_STABLE_EXT_SRC @@ -753,24 +716,6 @@ if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP") " in CUDA target architectures") endif() - # DeepSeek V3 fused A GEMM kernel (requires SM 9.0+, Hopper and later) - if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 13.0) - cuda_archs_loose_intersection(DSV3_FUSED_A_GEMM_ARCHS "9.0a;10.0f;10.7f;11.0f;12.0f" "${CUDA_ARCHS}") - else() - cuda_archs_loose_intersection(DSV3_FUSED_A_GEMM_ARCHS "9.0a;10.0a;10.1a;10.3a;12.0a;12.1a" "${CUDA_ARCHS}") - endif() - if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 12.0 AND DSV3_FUSED_A_GEMM_ARCHS) - set(DSV3_FUSED_A_GEMM_SRCS "csrc/dsv3_fused_a_gemm.cu") - set_gencode_flags_for_srcs( - SRCS "${DSV3_FUSED_A_GEMM_SRCS}" - CUDA_ARCHS "${DSV3_FUSED_A_GEMM_ARCHS}") - list(APPEND VLLM_STABLE_EXT_SRC "${DSV3_FUSED_A_GEMM_SRCS}") - message(STATUS "Building dsv3_fused_a_gemm for archs: ${DSV3_FUSED_A_GEMM_ARCHS}") - else() - message(STATUS "Not building dsv3_fused_a_gemm as no compatible archs found " - "in CUDA target architectures.") - endif() - # FP32 router GEMM (H=3072, E=256, M<=32). Requires SM90+ and CUDA >= 12.0. cuda_archs_sm90plus(FP32_ROUTER_GEMM_ARCHS "${CUDA_ARCHS}") if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 12.0 AND FP32_ROUTER_GEMM_ARCHS) @@ -1150,24 +1095,6 @@ if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP") "Building fused GDN decode for archs: ${FUSED_GDN_DECODE_ARCHS}") endif() - if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 13.0) - cuda_archs_loose_intersection(KIMI_K3_ATTN_RES_ARCHS - "10.0f" "${CUDA_ARCHS}") - endif() - if(KIMI_K3_ATTN_RES_ARCHS) - set(KIMI_K3_ATTN_RES_SRC - "csrc/kimi_k3/attn_res_kernel.cu") - set_gencode_flags_for_srcs( - SRCS "${KIMI_K3_ATTN_RES_SRC}" - CUDA_ARCHS "${KIMI_K3_ATTN_RES_ARCHS}") - set_property(SOURCE ${KIMI_K3_ATTN_RES_SRC} APPEND PROPERTY - COMPILE_OPTIONS - "$<$:--expt-relaxed-constexpr;--expt-extended-lambda;--use_fast_math>") - list(APPEND VLLM_STABLE_EXT_SRC "${KIMI_K3_ATTN_RES_SRC}") - message(STATUS - "Building Kimi K3 AttnRes for archs: ${KIMI_K3_ATTN_RES_ARCHS}") - endif() - # Hadacore kernels cuda_archs_loose_intersection(HADACORE_ARCHS "8.0+PTX;9.0+PTX" "${CUDA_ARCHS}") if(HADACORE_ARCHS) diff --git a/cmake/cpu_extension.cmake b/cmake/cpu_extension.cmake deleted file mode 100644 index c4d111e77b..0000000000 --- a/cmake/cpu_extension.cmake +++ /dev/null @@ -1,698 +0,0 @@ -include(FetchContent) -include(CheckCXXCompilerFlag) - -set(CMAKE_CXX_STANDARD_REQUIRED ON) -set(CMAKE_CXX_STANDARD 20) -set(CMAKE_CXX_EXTENSIONS ON) -set(CMAKE_EXPORT_COMPILE_COMMANDS ON) - -if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin") - set(MACOSX_FOUND TRUE) -endif() - - -# -# Define environment variables for special configurations -# -set(ENABLE_X86_ISA $ENV{VLLM_CPU_X86}) -set(ENABLE_ARM_BF16 $ENV{VLLM_CPU_ARM_BF16}) -set(ENABLE_ARM_I8MM $ENV{VLLM_CPU_ARM_I8MM}) -set(ENABLE_RVV_BF16 $ENV{VLLM_CPU_RVV_BF16}) - -include_directories("${CMAKE_SOURCE_DIR}/csrc") - -set (ENABLE_NUMA TRUE) - -# -# Check the compile flags -# -if(MACOSX_FOUND) - # Apple clang needs -Xpreprocessor to enable OpenMP. No runtime link is - # needed: _C is a dynamic_lookup bundle and resolves libomp from torch. - list(APPEND CXX_COMPILE_FLAGS - "-Xpreprocessor" "-fopenmp" - "-DVLLM_CPU_EXTENSION") -else() - list(APPEND CXX_COMPILE_FLAGS - "-fopenmp" - "-DVLLM_CPU_EXTENSION") - - # locate PyTorch's libgomp (e.g. site-packages/torch.libs/libgomp-947d5fa1.so.1.0.0) - # and create a local shim dir with it. When PyTorch is built from source or packaged - # by a distro (common on RISC-V, s390x, Fedora/RHEL aarch64), no vendored libgomp - # exists and the shim dir is empty; fall back to the system libgomp in that case. - vllm_prepare_torch_gomp_shim(VLLM_TORCH_GOMP_SHIM_DIR) - - if(VLLM_TORCH_GOMP_SHIM_DIR) - find_library(OPEN_MP - NAMES gomp - PATHS "${VLLM_TORCH_GOMP_SHIM_DIR}" - NO_DEFAULT_PATH - REQUIRED - ) - # Use the same libgomp as PyTorch at runtime - set(ENV{LD_LIBRARY_PATH} "${VLLM_TORCH_GOMP_SHIM_DIR}:$ENV{LD_LIBRARY_PATH}") - else() - # Fall back to system / toolchain libgomp - find_library(OPEN_MP NAMES gomp REQUIRED) - endif() -endif() - -if (NOT MACOSX_FOUND) - execute_process(COMMAND cat /proc/cpuinfo - RESULT_VARIABLE CPUINFO_RET - OUTPUT_VARIABLE CPUINFO) - if (NOT CPUINFO_RET EQUAL 0) - message(FATAL_ERROR "Failed to check CPU features via /proc/cpuinfo") - endif() -endif() - - -function (find_isa CPUINFO TARGET OUT) - string(FIND ${CPUINFO} ${TARGET} ISA_FOUND) - if(NOT ISA_FOUND EQUAL -1) - set(${OUT} ON PARENT_SCOPE) - else() - set(${OUT} OFF PARENT_SCOPE) - endif() -endfunction() - - -function(check_sysctl TARGET OUT) - execute_process(COMMAND sysctl -n "${TARGET}" - RESULT_VARIABLE SYSCTL_RET - OUTPUT_VARIABLE SYSCTL_INFO - ERROR_QUIET - OUTPUT_STRIP_TRAILING_WHITESPACE) - if(SYSCTL_RET EQUAL 0 AND - (SYSCTL_INFO STREQUAL "1" OR SYSCTL_INFO GREATER 0)) - set(${OUT} ON PARENT_SCOPE) - else() - set(${OUT} OFF PARENT_SCOPE) - endif() -endfunction() - -if (MACOSX_FOUND AND CMAKE_SYSTEM_PROCESSOR STREQUAL "arm64") - message(STATUS "Apple Silicon Detected") - set(APPLE_SILICON_FOUND TRUE) - set(ENABLE_NUMA OFF) - check_sysctl(hw.optional.neon ASIMD_FOUND) - check_sysctl(hw.optional.arm.FEAT_BF16 ARM_BF16_FOUND) - check_sysctl(hw.optional.arm.FEAT_I8MM ARM_I8MM_FOUND) -else() - find_isa(${CPUINFO} "Power11" POWER11_FOUND) - find_isa(${CPUINFO} "POWER10" POWER10_FOUND) - find_isa(${CPUINFO} "POWER9" POWER9_FOUND) - find_isa(${CPUINFO} "asimd" ASIMD_FOUND) # Check for ARM NEON support - find_isa(${CPUINFO} "bf16" ARM_BF16_FOUND) # Check for ARM BF16 support - find_isa(${CPUINFO} "i8mm" ARM_I8MM_FOUND) # Check for ARM I8MM support - find_isa(${CPUINFO} "S390" S390_FOUND) - find_isa(${CPUINFO} "zvfhmin" RVV_FP16_FOUND) # Check for RISC-V Vector FP16 support - find_isa(${CPUINFO} "zvfbfmin" RVV_BF16_FOUND) # Check for RISC-V Vector BF16 support - - # Support cross-compilation by allowing override via environment variables - if (ENABLE_ARM_BF16) - set(ARM_BF16_FOUND ON) - message(STATUS "ARM BF16 support enabled via VLLM_CPU_ARM_BF16 environment variable") - endif() - if (ENABLE_ARM_I8MM) - set(ARM_I8MM_FOUND ON) - message(STATUS - "ARM I8MM support enabled via VLLM_CPU_ARM_I8MM environment variable") - endif() - # Some kernels (e.g. Bianbu on Spacemit X100) do not report zvfbfmin - # in /proc/cpuinfo despite hardware support. VLLM_CPU_RVV_BF16=1 - # overrides the detection result. - if (ENABLE_RVV_BF16) - set(RVV_BF16_FOUND ON) - message(STATUS "RVV BF16 support enabled via VLLM_CPU_RVV_BF16 environment variable") - endif() -endif() - -if (CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|amd64" OR ENABLE_X86_ISA) - set(ENABLE_X86_ISA ON) - if (NOT (CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND - CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 12.3)) - message(FATAL_ERROR "X86 backend requires gcc/g++ >= 12.3") - endif() - # Work around a GCC 15 optimizer crash in oneDNN (tree-ssa-pre pass). - # Keep this scoped to GCC 15+ so older toolchains are unaffected. - if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND - CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 15) - list(APPEND CXX_COMPILE_FLAGS "-fno-tree-pre") - endif() - list(APPEND CXX_COMPILE_FLAGS "-mf16c") - list(APPEND CXX_COMPILE_FLAGS_AVX512 ${CXX_COMPILE_FLAGS}) - list(APPEND CXX_COMPILE_FLAGS_AVX2 ${CXX_COMPILE_FLAGS}) - list(APPEND CXX_COMPILE_FLAGS_AVX512 - "-mavx512f" - "-mavx512vl" - "-mavx512bw" - "-mavx512dq") - list(APPEND CXX_COMPILE_FLAGS_AVX512_AMX - ${CXX_COMPILE_FLAGS_AVX512} - "-mamx-bf16" - "-mamx-tile" - "-mavx512bf16" - "-mavx512vnni") - # Some packaged GCC 14 builds still do not accept -mamx-fp8, so gate on - # actual flag support rather than compiler version alone. - # Option to enable/disable AMX-FP8 support (default: ON) - option(ENABLE_AMX_FP8 "Enable AMX-FP8 support" ON) - - if (ENABLE_AMX_FP8) - check_cxx_compiler_flag("-mamx-fp8" COMPILER_SUPPORTS_AMX_FP8_FLAG) - if (COMPILER_SUPPORTS_AMX_FP8_FLAG) - list(APPEND CXX_COMPILE_FLAGS_AVX512_AMX "-mamx-fp8") - set(AMX_FP8_SUPPORTED TRUE) - else() - set(AMX_FP8_SUPPORTED FALSE) - message(STATUS "AMX-FP8 disabled: compiler does not support -mamx-fp8 (compiler: ${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION})") - endif() - else() - set(AMX_FP8_SUPPORTED FALSE) - message(STATUS "AMX-FP8 disabled by ENABLE_AMX_FP8=OFF") - endif() - - # AVX10.2 arrived in GCC 15, so the target attribute, the fp8 convert - # intrinsics and __builtin_cpu_supports("avx10.2") are all hard errors on - # the 12.3+ the backend otherwise supports. Probe instead of assuming. - check_cxx_compiler_flag("-mavx10.2" COMPILER_SUPPORTS_AVX10_2_FLAG) - if (COMPILER_SUPPORTS_AVX10_2_FLAG) - set(AVX10_2_SUPPORTED TRUE) - else() - set(AVX10_2_SUPPORTED FALSE) - message(STATUS "AVX10.2 disabled: compiler does not support -mavx10.2 (compiler: ${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION})") - endif() - - list(APPEND CXX_COMPILE_FLAGS_AVX2 - "-mavx2") -elseif (POWER9_FOUND OR POWER10_FOUND OR POWER11_FOUND) - message(STATUS "PowerPC detected") - if (POWER9_FOUND) - list(APPEND CXX_COMPILE_FLAGS - "-mvsx" - "-mcpu=power9" - "-mtune=power9") - elseif (POWER10_FOUND OR POWER11_FOUND) - list(APPEND CXX_COMPILE_FLAGS - "-mvsx" - "-mcpu=power10" - "-mtune=power10") - endif() - -elseif (ASIMD_FOUND) - message(STATUS "ARMv8 or later architecture detected") - if(ARM_BF16_FOUND) - message(STATUS "BF16 extension detected") - set(MARCH_FLAGS "-march=armv8.2-a+bf16+dotprod+fp16") - add_compile_definitions(ARM_BF16_SUPPORT) - else() - message(WARNING "BF16 functionality is not available") - set(MARCH_FLAGS "-march=armv8.2-a+dotprod+fp16") - endif() - if(ARM_I8MM_FOUND) - message(STATUS "I8MM extension detected") - string(APPEND MARCH_FLAGS "+i8mm") - add_compile_definitions(ARM_I8MM_SUPPORT) - endif() - list(APPEND CXX_COMPILE_FLAGS ${MARCH_FLAGS}) -elseif (S390_FOUND) - message(STATUS "S390 detected") - # Check for S390 VXE support - list(APPEND CXX_COMPILE_FLAGS - "-mvx" - "-mzvector" - "-march=native" - "-mtune=native") -elseif (CMAKE_SYSTEM_PROCESSOR MATCHES "riscv64") - message(STATUS "RISC-V detected") - if(DEFINED VLLM_RVV_VLEN AND VLLM_RVV_VLEN LESS 0) - message(FATAL_ERROR - "VLLM_RVV_VLEN must be zero or a positive integer; got '${VLLM_RVV_VLEN}'") - endif() - # VLLM_RVV_VLEN selects the target VLEN. Auto-detected from /proc/cpuinfo - # by default; set -DVLLM_RVV_VLEN=0 to force scalar RISC-V build. - # Override with -DVLLM_RVV_VLEN=128 or -DVLLM_RVV_VLEN=256 for RVV. - if(NOT DEFINED VLLM_RVV_VLEN) - # Auto-detect: find the largest zvlb in /proc/cpuinfo isa line. - # Skip when cross-compiling — /proc/cpuinfo describes the build host. - if(CMAKE_CROSSCOMPILING) - message(STATUS "Cross-compiling: skipping VLEN auto-detection from /proc/cpuinfo") - elseif(EXISTS /proc/cpuinfo) - file(READ /proc/cpuinfo _cpuinfo) - set(_best 0) - foreach(_n IN ITEMS 128 256 512 1024) - if(_cpuinfo MATCHES "zvl${_n}b") - set(_best ${_n}) - endif() - endforeach() - # Only VLEN=128 and VLEN=256 are supported by the RVV kernels. - if(_best GREATER 256) - message(WARNING - "Detected VLEN=${_best} but only 128/256 are supported; " - "clamping to 256") - set(_best 256) - endif() - if(_best GREATER 0) - set(VLLM_RVV_VLEN ${_best}) - endif() - endif() - # If auto-detect failed (no /proc/cpuinfo or no zvlb reported) - # but the compiler supports RVV, require explicit specification. - if(NOT DEFINED VLLM_RVV_VLEN AND (RVV_FP16_FOUND OR RVV_BF16_FOUND)) - message(FATAL_ERROR - "RISC-V RVV is available but VLEN could not be auto-detected. " - "Please specify VLEN explicitly via CMAKE_ARGS:\n" - " CMAKE_ARGS='-DVLLM_RVV_VLEN=128' (for VLEN=128 hardware)\n" - " CMAKE_ARGS='-DVLLM_RVV_VLEN=256' (for VLEN=256 hardware, e.g. Spacemit X100)") - endif() - endif() - if(VLLM_RVV_VLEN AND VLLM_RVV_VLEN GREATER 0) - message(STATUS "RISC-V RVV VLEN=${VLLM_RVV_VLEN}") - # Sources gate FP16/BF16 paths on the compiler-provided - # __riscv_zvfh / __riscv_zvfbfmin macros, which GCC and clang - # define automatically when those extensions appear in -march. - if(RVV_BF16_FOUND) - message(STATUS "BF16 extension detected") - set(MARCH_FLAGS -march=rv64gcv_zvfh_zfbfmin_zvfbfmin_zvl${VLLM_RVV_VLEN}b -mrvv-vector-bits=zvl -mabi=lp64d) - elseif(RVV_FP16_FOUND) - message(WARNING "BF16 functionality is not available.") - set(MARCH_FLAGS -march=rv64gcv_zvfh_zvl${VLLM_RVV_VLEN}b -mrvv-vector-bits=zvl -mabi=lp64d) - else() - message(STATUS "compile riscv with scalar (no FP16/BF16)") - set(MARCH_FLAGS -march=rv64gc) - endif() - else() - message(STATUS "compile riscv with scalar") - set(MARCH_FLAGS -march=rv64gc) - endif() - list(APPEND CXX_COMPILE_FLAGS ${MARCH_FLAGS}) -else() - message(FATAL_ERROR "vLLM CPU backend requires X86, Power9+ ISA, S390X ISA, ARMv8 or RISC-V support.") -endif() - - -# Build oneDNN for GEMM kernels -if (ENABLE_X86_ISA OR (ASIMD_FOUND AND NOT APPLE_SILICON_FOUND) OR POWER9_FOUND OR POWER10_FOUND OR POWER11_FOUND OR RVV_FP16_FOUND OR RVV_BF16_FOUND OR S390_FOUND) - # Fetch and build Arm Compute Library (ACL) as oneDNN's backend for AArch64 - # TODO [fadara01]: remove this once ACL can be fetched and built automatically as a dependency of oneDNN - set(ONEDNN_AARCH64_USE_ACL OFF CACHE BOOL "") - if(ASIMD_FOUND) - # Set number of parallel build processes - include(ProcessorCount) - ProcessorCount(NPROC) - if(NOT NPROC) - set(NPROC 4) - endif() - - # Fetch and populate ACL - if(DEFINED ENV{ACL_ROOT_DIR} AND IS_DIRECTORY "$ENV{ACL_ROOT_DIR}") - message(STATUS "Using ACL from specified source directory: $ENV{ACL_ROOT_DIR}") - else() - message(STATUS "Downloading Arm Compute Library (ACL) from GitHub") - FetchContent_Populate(arm_compute - SUBBUILD_DIR "${FETCHCONTENT_BASE_DIR}/arm_compute-subbuild" - SOURCE_DIR "${FETCHCONTENT_BASE_DIR}/arm_compute-src" - GIT_REPOSITORY https://github.com/ARM-software/ComputeLibrary.git - GIT_TAG v52.6.0 - GIT_SHALLOW TRUE - GIT_PROGRESS TRUE - ) - set(ENV{ACL_ROOT_DIR} "${arm_compute_SOURCE_DIR}") - set(ACL_LIB_DIR "$ENV{ACL_ROOT_DIR}/build") - endif() - - # Build ACL with CMake - set(_cmake_config_cmd - ${CMAKE_COMMAND} -G Ninja -B build - -DARM_COMPUTE_BUILD_SHARED_LIB=OFF - -DCMAKE_BUILD_TYPE=Release - -DARM_COMPUTE_ARCH=armv8.2-a - -DARM_COMPUTE_ENABLE_ASSERTS=OFF - -DARM_COMPUTE_ENABLE_CPPTHREADS=OFF - -DARM_COMPUTE_ENABLE_OPENMP=ON - -DARM_COMPUTE_ENABLE_WERROR=OFF - -DARM_COMPUTE_BUILD_EXAMPLES=OFF - -DARM_COMPUTE_BUILD_TESTING=OFF) - set(_cmake_build_cmd - ${CMAKE_COMMAND} --build build -- -j${NPROC} - ) - - execute_process( - COMMAND ${_cmake_config_cmd} - WORKING_DIRECTORY "$ENV{ACL_ROOT_DIR}" - ) - execute_process( - COMMAND ${_cmake_build_cmd} - WORKING_DIRECTORY "$ENV{ACL_ROOT_DIR}" - RESULT_VARIABLE _acl_rc - ) - - if(NOT _acl_rc EQUAL 0) - message(FATAL_ERROR "ACL SCons build failed (exit ${_acl_rc}).") - endif() - message(STATUS "Arm Compute Library (ACL) built successfully.") - - # VLLM/oneDNN settings for ACL - set(ONEDNN_AARCH64_USE_ACL ON CACHE BOOL "" FORCE) - add_compile_definitions(VLLM_USE_ACL) - endif() - - set(FETCHCONTENT_SOURCE_DIR_ONEDNN "$ENV{FETCHCONTENT_SOURCE_DIR_ONEDNN}" CACHE PATH "Path to a local oneDNN source directory.") - - if(FETCHCONTENT_SOURCE_DIR_ONEDNN) - message(STATUS "Using oneDNN from specified source directory: ${FETCHCONTENT_SOURCE_DIR_ONEDNN}") - FetchContent_Declare( - oneDNN - SOURCE_DIR ${FETCHCONTENT_SOURCE_DIR_ONEDNN} - ) - else() - message(STATUS "Downloading oneDNN from GitHub") - if(ASIMD_FOUND AND NOT APPLE_SILICON_FOUND) - message(STATUS "aarch64 detected: using pinned oneDNN commit 9c5be1cc59e368aebf0909e6cf20f981ea61462a") - FetchContent_Declare( - oneDNN - GIT_REPOSITORY https://github.com/oneapi-src/oneDNN.git - GIT_TAG 9c5be1cc59e368aebf0909e6cf20f981ea61462a - GIT_PROGRESS TRUE - GIT_SHALLOW FALSE - ) - else() - FetchContent_Declare( - oneDNN - GIT_REPOSITORY https://github.com/oneapi-src/oneDNN.git - GIT_TAG v3.13 - GIT_PROGRESS TRUE - GIT_SHALLOW TRUE - ) - endif() - endif() - - set(ONEDNN_LIBRARY_TYPE "STATIC") - set(ONEDNN_BUILD_DOC "OFF") - set(ONEDNN_BUILD_EXAMPLES "OFF") - set(ONEDNN_BUILD_TESTS "OFF") - set(ONEDNN_ENABLE_WORKLOAD "INFERENCE") - set(ONEDNN_ENABLE_PRIMITIVE "MATMUL;REORDER") - set(ONEDNN_BUILD_GRAPH "OFF") - set(ONEDNN_ENABLE_JIT_PROFILING "ON") - set(ONEDNN_ENABLE_ITT_TASKS "ON") - set(ONEDNN_ENABLE_MAX_CPU_ISA "ON") - set(ONEDNN_ENABLE_CPU_ISA_HINTS "ON") - set(ONEDNN_VERBOSE "ON") - set(CMAKE_POLICY_DEFAULT_CMP0077 NEW) - - set(VLLM_BUILD_TYPE ${CMAKE_BUILD_TYPE}) - set(CMAKE_BUILD_TYPE "Release") # remove oneDNN debug symbols to reduce size - - if(S390_FOUND) - FetchContent_GetProperties(oneDNN) - if(NOT onednn_POPULATED) - FetchContent_Populate(oneDNN) - # Patch s390x helpers.h: ALWAYS_INLINE on operator+= breaks C++20/GCC14 - file(READ "${onednn_SOURCE_DIR}/src/cpu/s390x/helpers.h" _helpers_content) - string(REPLACE - "vec_type_t &ALWAYS_INLINE operator+=" - "ALWAYS_INLINE vec_type_t &operator+=" - _helpers_content "${_helpers_content}") - file(WRITE "${onednn_SOURCE_DIR}/src/cpu/s390x/helpers.h" "${_helpers_content}") - add_subdirectory("${onednn_SOURCE_DIR}" "${onednn_BINARY_DIR}") - endif() - else() - FetchContent_MakeAvailable(oneDNN) - endif() - set(CMAKE_BUILD_TYPE ${VLLM_BUILD_TYPE}) - add_library(dnnl_ext OBJECT "csrc/cpu/dnnl_helper.cpp") - target_include_directories( - dnnl_ext - PUBLIC ${oneDNN_SOURCE_DIR}/include - PUBLIC ${oneDNN_BINARY_DIR}/include - PRIVATE ${oneDNN_SOURCE_DIR}/src - ) - target_link_libraries(dnnl_ext dnnl torch) - if (ENABLE_X86_ISA) - target_compile_options(dnnl_ext PRIVATE ${CXX_COMPILE_FLAGS_AVX2} -fPIC) - else() - target_compile_options(dnnl_ext PRIVATE ${CXX_COMPILE_FLAGS} -fPIC) - endif() - list(APPEND LIBS dnnl_ext) - - - set(USE_ONEDNN ON) -else() - set(USE_ONEDNN OFF) -endif() - -# TODO: Refactor this -if (ENABLE_X86_ISA) - message(STATUS "CPU extension (AVX512F + BF16 + VNNI + AMX) compile flags: ${CXX_COMPILE_FLAGS_AVX512_AMX}") - message(STATUS "CPU extension (AVX512F) compile flags: ${CXX_COMPILE_FLAGS_AVX512}") - message(STATUS "CPU extension (AVX2) compile flags: ${CXX_COMPILE_FLAGS_AVX2}") -else() - message(STATUS "CPU extension compile flags: ${CXX_COMPILE_FLAGS}") -endif() - -if(ENABLE_NUMA) - list(APPEND LIBS numa) -else() - message(STATUS "NUMA is disabled") - add_compile_definitions(-DVLLM_NUMA_DISABLED) -endif() - -# check if the pytorch wheel ships libopenblas.so. -set(VLLM_OPENBLAS_LIB "") -if (NOT ENABLE_X86_ISA) - file(GLOB _VLLM_TORCH_OPENBLAS_LIBS - "${TORCH_INSTALL_PREFIX}/lib/libopenblas*.so*") - # Note: we don't link openblas directly to _C extension, as it's available through libtorch.so - if (_VLLM_TORCH_OPENBLAS_LIBS) - list(GET _VLLM_TORCH_OPENBLAS_LIBS 0 VLLM_OPENBLAS_LIB) - message(STATUS "CPU OpenBLAS library: ${VLLM_OPENBLAS_LIB}") - endif() -endif() - -# -# Generate CPU attention dispatch header -# -message(STATUS "Generating CPU attention dispatch header") -execute_process( - COMMAND ${Python_EXECUTABLE} ${CMAKE_SOURCE_DIR}/csrc/cpu/generate_cpu_attn_dispatch.py - WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/csrc/cpu - RESULT_VARIABLE GEN_RESULT -) -if(NOT GEN_RESULT EQUAL 0) - message(FATAL_ERROR "Failed to generate CPU attention dispatch header") -endif() - -# -# _C extension -# -set(VLLM_EXT_SRC - "csrc/cpu/activation.cpp" - "csrc/cpu/sgl-kernels/fla.cpp" - "csrc/cpu/utils.cpp" - "csrc/cpu/spec_decode_utils.cpp" - "csrc/cpu/layernorm.cpp" - "csrc/cpu/mla_decode.cpp" - "csrc/cpu/pos_encoding.cpp" - "csrc/cpu/mamba_cpu.cpp" - "csrc/cpu/dynamic_4bit_int_moe_cpu.cpp" - "csrc/cpu/cpu_fused_moe.cpp" - "csrc/cpu/cpu_attn.cpp" - "csrc/cpu/cpu_isa.cpp" - "csrc/cpu/sampling_kernels.cpp" - "csrc/cpu/torch_bindings.cpp") - -if (CMAKE_SYSTEM_PROCESSOR MATCHES "riscv64" AND VLLM_RVV_VLEN AND - VLLM_RVV_VLEN GREATER 0 AND (RVV_FP16_FOUND OR RVV_BF16_FOUND)) - set(VLLM_EXT_SRC - "csrc/cpu/cpu_wna16.cpp" - ${VLLM_EXT_SRC}) -endif() - -if (S390_FOUND) - set(VLLM_EXT_SRC - "csrc/cpu/cpu_wna16.cpp" - ${VLLM_EXT_SRC}) -endif() - -if (ASIMD_FOUND AND NOT APPLE_SILICON_FOUND) - set(VLLM_EXT_SRC - "csrc/cpu/shm.cpp" - "csrc/cpu/activation_lut_bf16.cpp" - "csrc/cpu/cpu_tanhf_neon.hpp" - ${VLLM_EXT_SRC}) - if (ARM_BF16_FOUND) - set(VLLM_EXT_SRC "csrc/cpu/sgl-kernels/conv.cpp" ${VLLM_EXT_SRC}) - if (ARM_I8MM_FOUND) - set(VLLM_EXT_SRC "csrc/cpu/cpu_fused_moe_int8.cpp" ${VLLM_EXT_SRC}) - endif() - endif() -endif() - -if (POWER9_FOUND OR POWER10_FOUND OR POWER11_FOUND) - set(VLLM_EXT_SRC - "csrc/cpu/cpu_wna16.cpp" - "csrc/cpu/shm.cpp" - ${VLLM_EXT_SRC}) -endif() - -if (POWER10_FOUND OR POWER11_FOUND) - set(VLLM_EXT_SRC - "csrc/cpu/cpu_fused_moe_int8.cpp" - ${VLLM_EXT_SRC}) -endif() - -if(USE_ONEDNN) - set(VLLM_EXT_SRC - "csrc/cpu/dnnl_kernels.cpp" - ${VLLM_EXT_SRC}) -endif() - -if (CMAKE_SYSTEM_PROCESSOR MATCHES "riscv64") - set(VLLM_EXT_SRC - "csrc/cpu/sgl-kernels/gemm_int4.cpp" - ${VLLM_EXT_SRC}) -endif() - -if (ENABLE_X86_ISA) - set(VLLM_EXT_SRC_SGL - "csrc/cpu/sgl-kernels/conv.cpp" - "csrc/cpu/sgl-kernels/gemm.cpp" - "csrc/cpu/sgl-kernels/gemm_int8.cpp" - "csrc/cpu/sgl-kernels/gemm_fp8.cpp" - "csrc/cpu/sgl-kernels/gemm_fp8_w8a8.cpp" - "csrc/cpu/sgl-kernels/gemm_int4.cpp" - "csrc/cpu/sgl-kernels/moe.cpp" - "csrc/cpu/sgl-kernels/moe_int8.cpp" - "csrc/cpu/sgl-kernels/moe_int4.cpp" - "csrc/cpu/sgl-kernels/moe_fp8.cpp" - "csrc/cpu/sgl-kernels/moe_fp8_w8a8.cpp" - "csrc/cpu/sgl-kernels/bmm.cpp" - "csrc/cpu/sgl-kernels/decode.cpp" - "csrc/cpu/sgl-kernels/extend.cpp" - "csrc/cpu/sgl-kernels/mla_cache.cpp" - "csrc/cpu/sgl-kernels/mhc.cpp" - "csrc/cpu/sgl-kernels/store_cache.cpp" - "csrc/cpu/sgl-kernels/flash_mla.cpp" - "csrc/cpu/sgl-kernels/compressor.cpp" - "csrc/cpu/sgl-kernels/paged_mqa_logits.cpp" - "csrc/cpu/sgl-kernels/topk.cpp" - "csrc/cpu/sgl-kernels/indexer.cpp") - - set(VLLM_EXT_SRC_AVX512 - "csrc/cpu/sgl-kernels/fla.cpp" - "csrc/cpu/shm.cpp" - "csrc/cpu/cpu_wna16.cpp" - "csrc/cpu/cpu_fused_moe.cpp" - "csrc/cpu/utils.cpp" - "csrc/cpu/spec_decode_utils.cpp" - "csrc/cpu/cpu_attn.cpp" - "csrc/cpu/cpu_isa.cpp" - "csrc/cpu/dnnl_kernels.cpp" - "csrc/cpu/mamba_cpu.cpp" - "csrc/cpu/sampling_kernels.cpp" - "csrc/cpu/torch_bindings.cpp" - # TODO: Remove these files - "csrc/cpu/activation.cpp" - "csrc/cpu/layernorm.cpp" - "csrc/cpu/mla_decode.cpp" - "csrc/cpu/pos_encoding.cpp" - "csrc/cpu/dynamic_4bit_int_moe_cpu.cpp") - - set(VLLM_EXT_SRC_AVX2 - "csrc/cpu/sgl-kernels/fla.cpp" - "csrc/cpu/cpu_fused_moe.cpp" - "csrc/cpu/utils.cpp" - "csrc/cpu/spec_decode_utils.cpp" - "csrc/cpu/cpu_attn.cpp" - "csrc/cpu/cpu_isa.cpp" - "csrc/cpu/mamba_cpu.cpp" - "csrc/cpu/dnnl_kernels.cpp" - "csrc/cpu/sampling_kernels.cpp" - "csrc/cpu/torch_bindings.cpp" - # TODO: Remove these files - "csrc/cpu/activation.cpp" - "csrc/cpu/layernorm.cpp" - "csrc/cpu/mla_decode.cpp" - "csrc/cpu/pos_encoding.cpp" - "csrc/cpu/dynamic_4bit_int_moe_cpu.cpp") - - message(STATUS "CPU extension (AVX512F + BF16 + VNNI + AMX) source files: ${VLLM_EXT_SRC_AVX512} ${VLLM_EXT_SRC_SGL}") - message(STATUS "CPU extension (AVX512F) source files: ${VLLM_EXT_SRC_AVX512}") - message(STATUS "CPU extension (AVX2) source files: ${VLLM_EXT_SRC_AVX2}") - - set(_C_LIBS numa dnnl_ext) - set(_C_AVX512_LIBS numa dnnl_ext) - set(_C_AVX2_LIBS numa dnnl_ext) - - # AMX + AVX512F + AVX512BF16 + AVX512VNNI - define_extension_target( - _C - DESTINATION vllm - LANGUAGE CXX - SOURCES ${VLLM_EXT_SRC_AVX512} ${VLLM_EXT_SRC_SGL} - LIBRARIES ${_C_LIBS} - COMPILE_FLAGS ${CXX_COMPILE_FLAGS_AVX512_AMX} - USE_SABI 3 - WITH_SOABI - ) - - # For AMX kernels - target_compile_definitions(_C PRIVATE "-DCPU_CAPABILITY_AMXBF16") - if (AMX_FP8_SUPPORTED) - target_compile_definitions(_C PRIVATE "-DCPU_CAPABILITY_AMXFP8") - message(STATUS "AMX-FP8 (Diamond Rapids) enabled") - endif() - - # For the sgl-kernels AVX10.2 fp8 quantize paths - if (AVX10_2_SUPPORTED) - target_compile_definitions(_C PRIVATE "-DCPU_CAPABILITY_AVX10_2") - message(STATUS "AVX10.2 enabled") - endif() - - # AVX512F - define_extension_target( - _C_AVX512 - DESTINATION vllm - LANGUAGE CXX - SOURCES ${VLLM_EXT_SRC_AVX512} - LIBRARIES ${_C_AVX512_LIBS} - COMPILE_FLAGS ${CXX_COMPILE_FLAGS_AVX512} - USE_SABI 3 - WITH_SOABI - ) - - # AVX2 - define_extension_target( - _C_AVX2 - DESTINATION vllm - LANGUAGE CXX - SOURCES ${VLLM_EXT_SRC_AVX2} - LIBRARIES ${_C_AVX2_LIBS} - COMPILE_FLAGS ${CXX_COMPILE_FLAGS_AVX2} - USE_SABI 3 - WITH_SOABI - ) -else() - message(STATUS "CPU extension source files: ${VLLM_EXT_SRC}") - # - # Define extension targets - # - define_extension_target( - _C - DESTINATION vllm - LANGUAGE CXX - SOURCES ${VLLM_EXT_SRC} - LIBRARIES ${LIBS} - COMPILE_FLAGS ${CXX_COMPILE_FLAGS} - USE_SABI 3 - WITH_SOABI - ) - if (VLLM_OPENBLAS_LIB) - target_compile_definitions(_C PRIVATE VLLM_HAS_OPENBLAS) - endif() -endif() - -message(STATUS "Enabling C extension.") diff --git a/csrc/cpu/activation.cpp b/csrc/cpu/activation.cpp deleted file mode 100644 index 2f06813a19..0000000000 --- a/csrc/cpu/activation.cpp +++ /dev/null @@ -1,175 +0,0 @@ -#include "cpu_types.hpp" - -namespace { -template -void activation_kernel(int num_tokens, int d, scalar_t* __restrict__ input, - scalar_t* __restrict__ output) { - using scalar_vec_t = vec_op::vec_t; - constexpr int VEC_ELEM_NUM = scalar_vec_t::get_elem_num(); - - TORCH_CHECK(d % VEC_ELEM_NUM == 0); - -#pragma omp parallel for - for (int i = 0; i < num_tokens; ++i) { - for (int j = 0; j < d; j += VEC_ELEM_NUM) { - int start = i * d; - if constexpr (is_gated) { - start *= 2; - } - - const scalar_vec_t x(input + start + j); - const vec_op::FP32Vec8 f32_x(x); - vec_op::FP32Vec8 f32_ans = func(f32_x); - - if constexpr (is_gated) { - const scalar_vec_t y(input + start + d + j); - const vec_op::FP32Vec8 f32_y(y); - f32_ans = f32_y * f32_ans; - } - - const scalar_vec_t result(f32_ans); - result.save(output + i * d + j); - } - } -} - -FORCE_INLINE vec_op::FP32Vec8 silu_act(const vec_op::FP32Vec8& x) { - const vec_op::FP32Vec8 zeros(0.0); - const vec_op::FP32Vec8 ones(1.0); - return x / (ones + (zeros - x).exp()); -} - -FORCE_INLINE vec_op::FP32Vec8 gelu_new_act(const vec_op::FP32Vec8& x) { - const vec_op::FP32Vec8 ones(1.0); - const vec_op::FP32Vec8 w1(0.79788456f); - const vec_op::FP32Vec8 w2(0.044715f); - const vec_op::FP32Vec8 w3(0.5); - const vec_op::FP32Vec8 x3 = x * x * x; - const vec_op::FP32Vec8 t = (w1 * (x + w2 * x3)).tanh(); - return w3 * x * (ones + t); -} - -FORCE_INLINE vec_op::FP32Vec8 gelu_fast_act(const vec_op::FP32Vec8& x) { - const vec_op::FP32Vec8 ones(1.0); - const vec_op::FP32Vec8 w1(0.79788456f); - const vec_op::FP32Vec8 w2(0.044715f); - const vec_op::FP32Vec8 w3(0.5); - const vec_op::FP32Vec8 t = (x * w1 * (ones + x * w2 * x)).tanh(); - return w3 * x * (ones + t); -} - -FORCE_INLINE vec_op::FP32Vec8 gelu_quick_act(const vec_op::FP32Vec8& x) { - const vec_op::FP32Vec8 zeros(0.0); - const vec_op::FP32Vec8 ones(1.0); - const vec_op::FP32Vec8 w1(1.702f); - return x / (ones + (zeros - w1 * x).exp()); -} - -FORCE_INLINE vec_op::FP32Vec8 gelu_act(const vec_op::FP32Vec8& x) { - const vec_op::FP32Vec8 ones(1.0); - const vec_op::FP32Vec8 w1(M_SQRT1_2); - const vec_op::FP32Vec8 w2(0.5); - return x * w2 * (ones + (x * w1).er()); -} - -FORCE_INLINE vec_op::FP32Vec8 gelu_tanh_act(const vec_op::FP32Vec8& x) { - const vec_op::FP32Vec8 ones(1.0); - const vec_op::FP32Vec8 w1(M_SQRT2 * M_2_SQRTPI * 0.5); - const vec_op::FP32Vec8 w2(0.5); - const vec_op::FP32Vec8 w3(0.044715); - const vec_op::FP32Vec8 x_3 = x * x * x; - const vec_op::FP32Vec8 inner = w1 * (x + x_3 * w3); - return x * w2 * (ones + inner.tanh()); -} -}; // namespace - -void silu_and_mul(torch::Tensor& out, torch::Tensor& input) { - int num_tokens = input.numel() / input.size(-1); - int d = input.size(-1) / 2; - - VLLM_DISPATCH_FLOATING_TYPES(input.scalar_type(), "silu_and_mul_impl", [&] { - CPU_KERNEL_GUARD_IN(silu_and_mul_impl) - activation_kernel( - num_tokens, d, input.data_ptr(), out.data_ptr()); - CPU_KERNEL_GUARD_OUT(silu_and_mul_impl) - }); -} - -void gelu_and_mul(torch::Tensor& out, // [..., d] - torch::Tensor& input) // [..., 2 * d] -{ - int num_tokens = input.numel() / input.size(-1); - int d = input.size(-1) / 2; - - VLLM_DISPATCH_FLOATING_TYPES(input.scalar_type(), "gelu_and_mul_impl", [&] { - CPU_KERNEL_GUARD_IN(gelu_and_mul_impl) - activation_kernel( - num_tokens, d, input.data_ptr(), out.data_ptr()); - CPU_KERNEL_GUARD_OUT(gelu_and_mul_impl) - }); -} - -void gelu_tanh_and_mul(torch::Tensor& out, // [..., d] - torch::Tensor& input) // [..., 2 * d] -{ - int num_tokens = input.numel() / input.size(-1); - int d = input.size(-1) / 2; - - VLLM_DISPATCH_FLOATING_TYPES( - input.scalar_type(), "gelu_tanh_and_mul_impl", [&] { - CPU_KERNEL_GUARD_IN(gelu_tanh_and_mul_impl) - activation_kernel( - num_tokens, d, input.data_ptr(), - out.data_ptr()); - CPU_KERNEL_GUARD_OUT(gelu_tanh_and_mul_impl) - }); -} - -void gelu_tanh(torch::Tensor& out, torch::Tensor& input) { - int num_tokens = input.numel() / input.size(-1); - int d = input.size(-1); - - VLLM_DISPATCH_FLOATING_TYPES(input.scalar_type(), "gelu_tanh_impl", [&] { - CPU_KERNEL_GUARD_IN(gelu_tanh_impl) - activation_kernel( - num_tokens, d, input.data_ptr(), out.data_ptr()); - CPU_KERNEL_GUARD_OUT(gelu_tanh_impl) - }); -} - -void gelu_new(torch::Tensor& out, torch::Tensor& input) { - int num_tokens = input.numel() / input.size(-1); - int d = input.size(-1); - - VLLM_DISPATCH_FLOATING_TYPES(input.scalar_type(), "gelu_new_impl", [&] { - CPU_KERNEL_GUARD_IN(gelu_new_impl) - activation_kernel( - num_tokens, d, input.data_ptr(), out.data_ptr()); - CPU_KERNEL_GUARD_OUT(gelu_new_impl) - }); -} - -void gelu_fast(torch::Tensor& out, torch::Tensor& input) { - int num_tokens = input.numel() / input.size(-1); - int d = input.size(-1); - - VLLM_DISPATCH_FLOATING_TYPES(input.scalar_type(), "gelu_fast_impl", [&] { - CPU_KERNEL_GUARD_IN(gelu_fast_impl) - activation_kernel( - num_tokens, d, input.data_ptr(), out.data_ptr()); - CPU_KERNEL_GUARD_OUT(gelu_fast_impl) - }); -} - -void gelu_quick(torch::Tensor& out, torch::Tensor& input) { - int num_tokens = input.numel() / input.size(-1); - int d = input.size(-1); - - VLLM_DISPATCH_FLOATING_TYPES(input.scalar_type(), "gelu_quick_impl", [&] { - CPU_KERNEL_GUARD_IN(gelu_quick_impl) - activation_kernel( - num_tokens, d, input.data_ptr(), out.data_ptr()); - CPU_KERNEL_GUARD_OUT(gelu_quick_impl) - }); -} diff --git a/csrc/cpu/activation_lut_bf16.cpp b/csrc/cpu/activation_lut_bf16.cpp deleted file mode 100644 index 0ff2567e1e..0000000000 --- a/csrc/cpu/activation_lut_bf16.cpp +++ /dev/null @@ -1,71 +0,0 @@ -#include "cpu_types.hpp" - -#include -#include -#include -#include - -#include -#include -#include - -constexpr uint32_t ActivationLutSize = 1u << 16; - -at::Tensor gelu_reference(const at::Tensor& x) { return at::gelu(x, "none"); } - -void maybe_init_activation_lut_bf16( - uint16_t* lut, std::once_flag& once, - at::Tensor (*activation)(const at::Tensor&)) { - std::call_once(once, [&]() { - auto lut_input = - at::empty({static_cast(ActivationLutSize)}, - at::TensorOptions().device(at::kCPU).dtype(at::kFloat)); - auto* lut_input_ptr = lut_input.data_ptr(); -#pragma omp parallel for - for (uint32_t i = 0; i < ActivationLutSize; ++i) { - lut_input_ptr[i] = c10::detail::f32_from_bits(static_cast(i)); - } - - auto lut_output = activation(lut_input); - const auto* lut_output_ptr = lut_output.data_ptr(); -#pragma omp parallel for - for (uint32_t i = 0; i < ActivationLutSize; ++i) { - lut[i] = c10::detail::round_to_nearest_even(lut_output_ptr[i]); - } - }); -} - -void activation_lut_bf16(torch::Tensor& out, torch::Tensor& input, - const uint16_t* lut, const char* op_name) { - TORCH_CHECK(input.scalar_type() == at::kBFloat16, op_name, - ": input must be bfloat16"); - TORCH_CHECK(out.scalar_type() == at::kBFloat16, op_name, - ": out must be bfloat16"); - TORCH_CHECK(input.is_contiguous(), op_name, ": input must be contiguous"); - TORCH_CHECK(out.is_contiguous(), op_name, ": out must be contiguous"); - - const auto* src = - reinterpret_cast(input.data_ptr()); - auto* dst = reinterpret_cast(out.data_ptr()); - const int64_t n = input.numel(); - - CPU_KERNEL_GUARD_IN(activation_lut_bf16_impl) -#pragma omp parallel for - for (int64_t i = 0; i < n; ++i) { - dst[i] = lut[src[i]]; - } - CPU_KERNEL_GUARD_OUT(activation_lut_bf16_impl) -} - -void activation_lut_bf16(torch::Tensor& out, torch::Tensor& input, - const std::string& activation) { - if (activation == "gelu") { - static std::array lut{}; - static std::once_flag once; - maybe_init_activation_lut_bf16(lut.data(), once, gelu_reference); - activation_lut_bf16(out, input, lut.data(), "gelu_lut"); - return; - } - - TORCH_CHECK(false, "Unsupported activation: ", activation); -} diff --git a/csrc/cpu/cpu_arch_macros.h b/csrc/cpu/cpu_arch_macros.h deleted file mode 100644 index ccec4c2e87..0000000000 --- a/csrc/cpu/cpu_arch_macros.h +++ /dev/null @@ -1,197 +0,0 @@ -#ifndef CPU_ARCH_MACROS_H -#define CPU_ARCH_MACROS_H - -// x86_64 -#ifdef __x86_64__ - #define FAST_SPINNING _mm_pause(); - - #ifdef __AVX512F__ - #define DEFINE_FAST_EXP \ - const __m512 vec_factorial_1 = _mm512_set1_ps(0.999999701f); \ - const __m512 vec_factorial_2 = _mm512_set1_ps(0.499991506f); \ - const __m512 vec_factorial_3 = _mm512_set1_ps(0.166676521f); \ - const __m512 vec_factorial_4 = _mm512_set1_ps(0.0418978221f); \ - const __m512 vec_factorial_5 = _mm512_set1_ps(0.00828929059f); \ - const __m512 vec_exp_log2ef = \ - _mm512_castsi512_ps(_mm512_set1_epi32(0x3fb8aa3b)); \ - const __m512 vec_half = _mm512_set1_ps(0.5f); \ - const __m512 vec_one = _mm512_set1_ps(1.f); \ - const __m512 vec_zero = _mm512_set1_ps(0.f); \ - const __m512 vec_two = _mm512_set1_ps(2.f); \ - const __m512 vec_ln2f = \ - _mm512_castsi512_ps(_mm512_set1_epi32(0x3f317218)); \ - const __m512 vec_ln_flt_min = \ - _mm512_castsi512_ps(_mm512_set1_epi32(0xc2aeac50)); \ - const __m512 vec_ln_flt_max = \ - _mm512_castsi512_ps(_mm512_set1_epi32(0x42b17218)); \ - const __m512i vec_127 = _mm512_set1_epi32(0x0000007f); \ - const int n_mantissa_bits = 23; \ - auto fast_exp = [&](const vec_op::FP32Vec16& vec) __attribute__(( \ - always_inline)) { \ - __m512 values = vec.reg; \ - auto less_ln_flt_min_mask = \ - _mm512_cmp_ps_mask(values, vec_ln_flt_min, 1 /*_CMP_LT_OS*/); \ - auto vec_src = _mm512_min_ps(values, vec_ln_flt_max); \ - vec_src = _mm512_max_ps(vec_src, vec_ln_flt_min); \ - auto vec_fx = _mm512_fmadd_ps(vec_src, vec_exp_log2ef, vec_half); \ - auto vec_fx_i = _mm512_cvt_roundps_epi32( \ - vec_fx, _MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC); \ - vec_fx = _mm512_cvtepi32_ps(vec_fx_i); \ - auto vec_exp_poly = _mm512_fnmadd_ps(vec_fx, vec_ln2f, vec_src); \ - auto vec_res = \ - _mm512_fmadd_ps(vec_exp_poly, vec_factorial_5, vec_factorial_4); \ - vec_res = _mm512_fmadd_ps(vec_exp_poly, vec_res, vec_factorial_3); \ - vec_res = _mm512_fmadd_ps(vec_exp_poly, vec_res, vec_factorial_2); \ - vec_res = _mm512_fmadd_ps(vec_exp_poly, vec_res, vec_factorial_1); \ - vec_res = _mm512_fmadd_ps(vec_exp_poly, vec_res, vec_one); \ - auto vec_exp_number = _mm512_sub_ps(vec_fx, vec_one); \ - auto vec_exp_number_i = _mm512_cvtps_epi32(vec_exp_number); \ - auto vec_two_pow_n_i = _mm512_add_epi32(vec_exp_number_i, vec_127); \ - vec_two_pow_n_i = _mm512_slli_epi32(vec_two_pow_n_i, n_mantissa_bits); \ - auto vec_two_pow_n = _mm512_castsi512_ps(vec_two_pow_n_i); \ - vec_two_pow_n = _mm512_mask_blend_ps(less_ln_flt_min_mask, \ - vec_two_pow_n, vec_zero); \ - vec_res = _mm512_mul_ps(vec_res, vec_two_pow_n); \ - vec_res = _mm512_mul_ps(vec_res, vec_two); \ - vec_op::FP32Vec16 res(vec_res); \ - return res; \ - }; - #endif - -#endif - -#ifdef __aarch64__ - // Implementation of neon_expf copied from Arm Optimized Routines (expf - // AdvSIMD) - // https://github.com/ARM-software/optimized-routines/blob/master/math/aarch64/advsimd/expf.c - // - // Additional fast exponential intended for cases where outputs will be - // downcasted to FP16 / BF16 (e.g. attention softmax). Accurate within 1 ULP - // for FP16 Accurate within 1 ULP for BF16 for inputs in [-87.683, 88.376] & - // clamps inputs outside this range to 0 / inf. Implementation is similar to - // exp_u20, but: - // - uses a third degree polynomial approximation for exp(r) instead of a - // fifth degree one, with coefficients re-tuned. - // - does not split natural log (ln) into high / low parts - // - clamps exp(x) to 0 for x < -87.683113f and inf for x > 88.3762589f - // exp(x) = 2^n (exp(r)) - // r = x - n*ln2, with n = round(x/ln2) - // exp(r) ~ poly(r) = 1 + r + r^2 * (c3 + c2 * r) - // n = round(x / ln2), r = x - n*ln2 - #include - #define DEFINE_FAST_EXP \ - const float32x4_t inv_ln2 = vdupq_n_f32(0x1.715476p+0f); \ - const float ln2_hi = 0x1.62e4p-1f; \ - const float ln2_lo = 0x1.7f7d1cp-20f; \ - const float c0 = 0x1.0e4020p-7f; \ - const float c2 = 0x1.555e66p-3f; \ - const float32x4_t ln2_c02 = {ln2_hi, ln2_lo, c0, c2}; \ - const uint32x4_t exponent_bias = vdupq_n_u32(0x3f800000); \ - const float32x4_t c1 = vdupq_n_f32(0x1.573e2ep-5f); \ - const float32x4_t c3 = vdupq_n_f32(0x1.fffdb6p-2f); \ - const float32x4_t c4 = vdupq_n_f32(0x1.ffffecp-1f); \ - const float32x4_t pos_special_bound = vdupq_n_f32(0x1.5d5e2ap+6f); \ - const float32x4_t neg_special_bound = vnegq_f32(pos_special_bound); \ - const float32x4_t inf = \ - vdupq_n_f32(std::numeric_limits::infinity()); \ - const float32x4_t zero = vdupq_n_f32(0.0f); \ - auto neon_expf = [&](float32x4_t values) __attribute__((always_inline)) { \ - float32x4_t n = vrndaq_f32(vmulq_f32(values, inv_ln2)); \ - float32x4_t r = vfmsq_laneq_f32(values, n, ln2_c02, 0); \ - r = vfmsq_laneq_f32(r, n, ln2_c02, 1); \ - uint32x4_t e = vshlq_n_u32(vreinterpretq_u32_s32(vcvtq_s32_f32(n)), 23); \ - float32x4_t scale = vreinterpretq_f32_u32(vaddq_u32(e, exponent_bias)); \ - float32x4_t r2 = vmulq_f32(r, r); \ - float32x4_t p = vfmaq_laneq_f32(c1, r, ln2_c02, 2); \ - float32x4_t q = vfmaq_laneq_f32(c3, r, ln2_c02, 3); \ - q = vfmaq_f32(q, p, r2); \ - p = vmulq_f32(c4, r); \ - float32x4_t poly = vfmaq_f32(p, q, r2); \ - poly = vfmaq_f32(scale, poly, scale); \ - const uint32x4_t hi_mask = vcgeq_f32(values, pos_special_bound); \ - const uint32x4_t lo_mask = vcleq_f32(values, neg_special_bound); \ - poly = vbslq_f32(hi_mask, inf, poly); \ - return vbslq_f32(lo_mask, zero, poly); \ - }; \ - auto fast_exp = [&](const vec_op::FP32Vec16& vec) \ - __attribute__((always_inline)) { \ - float32x4x4_t result; \ - result.val[0] = neon_expf(vec.reg.val[0]); \ - result.val[1] = neon_expf(vec.reg.val[1]); \ - result.val[2] = neon_expf(vec.reg.val[2]); \ - result.val[3] = neon_expf(vec.reg.val[3]); \ - return vec_op::FP32Vec16(result); \ - }; \ - const float32x4_t lower_bound = vdupq_n_f32(-0x1.5ebb82p+6f); \ - const float32x4_t upper_bound = vdupq_n_f32(0x1.61814ap+6f); \ - constexpr float ln2 = 0x1.62e43p-1f; \ - constexpr float f_c2 = 0x1.5592ecp-3f; \ - const float32x4_t f_c3 = vdupq_n_f32(0x1.017d34p-1f); \ - auto neon_expf_f16 = [&](float32x4_t values) __attribute__(( \ - always_inline)) { \ - const uint32x4_t lt_lower = vcltq_f32(values, lower_bound); \ - const uint32x4_t gt_upper = vcgtq_f32(values, upper_bound); \ - float32x4_t n = vrndaq_f32(vmulq_f32(values, inv_ln2)); \ - float32x4_t r = vfmsq_n_f32(values, n, ln2); \ - uint32x4_t e = vshlq_n_u32(vreinterpretq_u32_s32(vcvtq_s32_f32(n)), 23); \ - float32x4_t r2 = vmulq_f32(r, r); \ - float32x4_t q = vfmaq_n_f32(f_c3, r, f_c2); \ - float32x4_t s = vaddq_f32(vdupq_n_f32(1.0f), r); \ - float32x4_t p = vfmaq_f32(s, q, r2); \ - float32x4_t y = \ - vreinterpretq_f32_u32(vaddq_u32(vreinterpretq_u32_f32(p), e)); \ - y = vbslq_f32(lt_lower, vdupq_n_f32(0.0f), y); \ - y = vbslq_f32(gt_upper, vdupq_n_f32(INFINITY), y); \ - return y; \ - }; \ - auto fast_exp_f16 = [&](const vec_op::FP32Vec16& vec) \ - __attribute__((always_inline)) { \ - float32x4x4_t result; \ - result.val[0] = neon_expf_f16(vec.reg.val[0]); \ - result.val[1] = neon_expf_f16(vec.reg.val[1]); \ - result.val[2] = neon_expf_f16(vec.reg.val[2]); \ - result.val[3] = neon_expf_f16(vec.reg.val[3]); \ - return vec_op::FP32Vec16(result); \ - }; - -#endif // __aarch64__ - -// RISC-V RVV -#ifdef __riscv_v - #include - - #ifdef __riscv_zihintpause - #define FAST_SPINNING __riscv_pause(); - #endif - - // FP32Vec16::exp() in cpu_types_riscv.hpp already implements the full - // polynomial approximation for RVV, so we simply delegate to it. - #define DEFINE_FAST_EXP \ - auto fast_exp = [&](const vec_op::FP32Vec16& vec) \ - __attribute__((always_inline)) { return vec.exp(); }; - -#endif // __riscv_v - -// Power VSX -#ifdef __powerpc__ - // FP32Vec16::exp() in cpu_types_vsx.hpp delegates to FP32Vec8::exp(), which - // implements a vectorised 5-term minimax polynomial using VSX intrinsics. - #define DEFINE_FAST_EXP \ - auto fast_exp = [&](const vec_op::FP32Vec16& vec) \ - __attribute__((always_inline)) { return vec.exp(); }; \ - auto fast_exp_f16 = fast_exp; - -#endif // __powerpc__ - -// IBM Z (s390x) VXE -#ifdef __s390x__ - // FP32Vec16::exp() in cpu_types_vxe.hpp delegates to FP32Vec8::exp(), which - // implements a vectorised 5-term minimax polynomial using VXE intrinsics. - #define DEFINE_FAST_EXP \ - auto fast_exp = [&](const vec_op::FP32Vec16& vec) \ - __attribute__((always_inline)) { return vec.exp(); }; \ - auto fast_exp_f16 = fast_exp; - -#endif // __s390x__ - -#endif diff --git a/csrc/cpu/cpu_attn.cpp b/csrc/cpu/cpu_attn.cpp deleted file mode 100644 index afdc0e0eca..0000000000 --- a/csrc/cpu/cpu_attn.cpp +++ /dev/null @@ -1,287 +0,0 @@ -#include "cpu_attn_dispatch_generated.h" - -// Runtime check for AMX-FP8, implemented in cpu_isa.cpp. -extern bool runtime_has_amx_fp8(); - -// Maps kv_cache_dtype string to Fp8KVCacheDataType enum. -// "auto" -> kAuto(0); "fp8"/"fp8_e4m3" -> kFp8E4M3; "fp8_e5m2" -> kFp8E5M2. -static inline cpu_attention::Fp8KVCacheDataType parse_fp8_kv_dtype( - const std::string& kv_cache_dtype) { - if (kv_cache_dtype == "fp8_e5m2") - return cpu_attention::Fp8KVCacheDataType::kFp8E5M2; - if (kv_cache_dtype == "fp8_e4m3" || kv_cache_dtype == "fp8") - return cpu_attention::Fp8KVCacheDataType::kFp8E4M3; - return cpu_attention::Fp8KVCacheDataType::kAuto; -} - -bool cpu_attn_has_isa(const std::string& isa) { - if (isa == "rvv") { -#if defined(__riscv) && defined(__riscv_v_min_vlen) && \ - (__riscv_v_min_vlen == 128 || __riscv_v_min_vlen == 256) - return true; -#else - return false; -#endif - } - if (isa == "amx_fp8") { -#ifdef CPU_CAPABILITY_AMXFP8 - // Guard with runtime CPUID check: the binary may be compiled with - // -mamx-fp8 (CPU_CAPABILITY_AMXFP8 defined) but run on a CPU that - // has AMX-BF16 without AMX-FP8 (e.g. Sapphire Rapids, Emerald Rapids). - return runtime_has_amx_fp8(); -#else - return false; -#endif - } - return false; -} - -torch::Tensor get_scheduler_metadata( - const int64_t num_req, const int64_t num_heads_q, - const int64_t num_heads_kv, const int64_t head_dim, - const torch::Tensor& seq_lens, at::ScalarType dtype, - const torch::Tensor& query_start_loc, const bool causal, - const int64_t window_size, const std::string& isa_hint, - const bool enable_kv_split, - const std::optional& dynamic_causal, - const std::string& kv_cache_dtype) { - cpu_attention::ISA isa; - if (isa_hint == "amx") { - isa = cpu_attention::ISA::AMX; - } else if (isa_hint == "amx_fp8") { - isa = cpu_attention::ISA::AMX_FP8; - } else if (isa_hint == "vec") { - isa = cpu_attention::ISA::VEC; - } else if (isa_hint == "vec16") { - isa = cpu_attention::ISA::VEC16; - } else if (isa_hint == "neon") { - isa = cpu_attention::ISA::NEON; - } else if (isa_hint == "vxe") { - isa = cpu_attention::ISA::VXE; - } else if (isa_hint == "rvv") { - isa = cpu_attention::ISA::RVV; - } else if (isa_hint == "vsx") { - isa = cpu_attention::ISA::VSX; - } else { - TORCH_CHECK(false, "Unsupported CPU attention ISA hint: " + isa_hint); - } - - cpu_attention::AttentionScheduler::ScheduleInput input; - input.num_reqs = num_req; - input.num_heads_q = num_heads_q; - input.num_heads_kv = num_heads_kv; - input.head_dim = head_dim; - input.query_start_loc = query_start_loc.data_ptr(); - input.seq_lens = seq_lens.data_ptr(); - - input.sliding_window_size = window_size; - input.causal = causal; - input.isa = isa; - input.enable_kv_split = enable_kv_split; - input.dynamic_causal = dynamic_causal.has_value() - ? dynamic_causal->data_ptr() - : nullptr; - - const int64_t kv_cache_idx = - static_cast(parse_fp8_kv_dtype(kv_cache_dtype)); - VLLM_DISPATCH_FLOATING_TYPES(dtype, "get_scheduler_metadata", [&]() { - CPU_ATTN_DISPATCH(head_dim, isa, kv_cache_idx, [&]() { - input.elem_size = sizeof(attn_impl::kv_cache_t); - input.q_buffer_elem_size = sizeof(attn_impl::q_buffer_t); - input.logits_buffer_elem_size = sizeof(attn_impl::logits_buffer_t); - input.output_buffer_elem_size = - sizeof(attn_impl::partial_output_buffer_t); - input.max_num_q_per_iter = attn_impl::MaxQHeadNumPerIteration; - input.kv_block_alignment = attn_impl::BlockSizeAlignment; - }); - }); - - cpu_attention::AttentionScheduler scheduler; - torch::Tensor metadata = scheduler.schedule(input); - return metadata; -} - -void cpu_attn_reshape_and_cache( - const torch::Tensor& key, // [token_num, head_num, head_size] - const torch::Tensor& value, // [token_num, head_num, head_size] - torch::Tensor& - key_cache, // [num_blocks, num_kv_heads, block_size, head_size] - torch::Tensor& - value_cache, // [num_blocks, num_kv_heads, block_size, head_size] - const torch::Tensor& slot_mapping, const std::string& isa, - const double k_scale = 1.0, const double v_scale = 1.0, - const std::string& kv_cache_dtype = "auto") { - TORCH_CHECK_EQ(key.dim(), 3); - TORCH_CHECK_EQ(value.dim(), 3); - TORCH_CHECK_EQ(key_cache.dim(), 4); - TORCH_CHECK_EQ(value_cache.dim(), 4); - TORCH_CHECK_EQ(key.stride(2), 1); - TORCH_CHECK_EQ(value.stride(2), 1); - - const int64_t kv_cache_idx = - static_cast(parse_fp8_kv_dtype(kv_cache_dtype)); - const bool is_fp8 = (kv_cache_idx != 0); - - if (is_fp8) { - TORCH_CHECK(key_cache.scalar_type() == at::ScalarType::Byte, - "key_cache must be uint8 for FP8 path"); - TORCH_CHECK(value_cache.scalar_type() == at::ScalarType::Byte, - "value_cache must be uint8 for FP8 path"); - TORCH_CHECK(k_scale > 0, "k_scale must be positive for FP8 path"); - TORCH_CHECK(v_scale > 0, "v_scale must be positive for FP8 path"); - } - - const float k_inv = is_fp8 ? 1.0f / static_cast(k_scale) : 0.0f; - const float v_inv = is_fp8 ? 1.0f / static_cast(v_scale) : 0.0f; - - const int64_t token_num = key.size(0); - const int64_t head_num = key.size(1); - const int64_t head_dim = key.size(2); - const int64_t num_blocks = key_cache.size(0); - const int64_t num_blocks_stride = key_cache.stride(0); - const int64_t cache_head_num_stride = key_cache.stride(1); - const int64_t block_size = key_cache.size(2); - const int64_t block_size_stride = key_cache.stride(2); - - cpu_attention::ISA isa_tag = [&]() { - if (isa == "amx") { - return cpu_attention::ISA::AMX; - } else if (isa == "amx_fp8") { - return cpu_attention::ISA::AMX_FP8; - } else if (isa == "vec") { - return cpu_attention::ISA::VEC; - } else if (isa == "vec16") { - return cpu_attention::ISA::VEC16; - } else if (isa == "neon") { - return cpu_attention::ISA::NEON; - } else if (isa == "vxe") { - return cpu_attention::ISA::VXE; - } else if (isa == "rvv") { - return cpu_attention::ISA::RVV; - } else if (isa == "vsx") { - return cpu_attention::ISA::VSX; - } else { - TORCH_CHECK(false, "Invalid ISA type: " + isa); - } - }(); - - if (is_fp8) { - TORCH_CHECK(isa_tag == cpu_attention::ISA::AMX || - isa_tag == cpu_attention::ISA::AMX_FP8 || - isa_tag == cpu_attention::ISA::VEC, - "FP8 KV cache is only supported on x86 (AMX_FP8/AMX/VEC) ISA"); - } - - VLLM_DISPATCH_FLOATING_TYPES( - key.scalar_type(), "cpu_attn_reshape_and_cache", [&]() { - CPU_ATTN_DISPATCH(head_dim, isa_tag, kv_cache_idx, [&]() { - using kv_t = typename attn_impl::kv_cache_t; - attn_impl::reshape_and_cache( - key.data_ptr(), value.data_ptr(), - reinterpret_cast(key_cache.data_ptr()), - reinterpret_cast(value_cache.data_ptr()), - slot_mapping.data_ptr(), token_num, key.stride(0), - value.stride(0), head_num, key.stride(1), value.stride(1), - num_blocks, num_blocks_stride, cache_head_num_stride, block_size, - block_size_stride, k_inv, v_inv); - }); - }); -} - -void cpu_attention_with_kv_cache( - const torch::Tensor& query, // [num_tokens, num_heads, head_size] - const torch::Tensor& - key_cache, // [num_blocks, num_kv_heads, block_size, head_size] - const torch::Tensor& - value_cache, // [num_blocks, num_kv_heads, block_size, head_size] - torch::Tensor& output, // [num_tokens, num_heads, head_size] - const torch::Tensor& query_start_loc, // [num_tokens + 1] - const torch::Tensor& seq_lens, // [num_tokens] - const double scale, const bool causal, - const std::optional& alibi_slopes, // [num_heads] - const int64_t sliding_window, - const torch::Tensor& block_table, // [num_tokens, max_block_num] - const double softcap, const torch::Tensor& scheduler_metadata, - const std::optional& s_aux, // [num_heads] - const std::optional& dynamic_causal, // [num_reqs] - const double k_scale = 1.0, const double v_scale = 1.0, - const std::string& kv_cache_dtype = "auto") { - TORCH_CHECK_EQ(query.dim(), 3); - TORCH_CHECK_EQ(query.stride(2), 1); - TORCH_CHECK_EQ(key_cache.dim(), 4); - TORCH_CHECK_EQ(value_cache.dim(), 4); - - const int64_t kv_cache_idx = - static_cast(parse_fp8_kv_dtype(kv_cache_dtype)); - const bool is_fp8 = (kv_cache_idx != 0); - if (is_fp8) { - TORCH_CHECK(key_cache.scalar_type() == at::ScalarType::Byte, - "key_cache must be uint8 for FP8 path"); - TORCH_CHECK(value_cache.scalar_type() == at::ScalarType::Byte, - "value_cache must be uint8 for FP8 path"); - TORCH_CHECK(k_scale > 0, "k_scale must be positive for FP8 path"); - TORCH_CHECK(v_scale > 0, "v_scale must be positive for FP8 path"); - } - - cpu_attention::AttentionInput input; - input.metadata = reinterpret_cast( - scheduler_metadata.data_ptr()); - input.num_tokens = query.size(0); - input.num_heads = query.size(1); - input.num_kv_heads = key_cache.size(1); - input.block_size = key_cache.size(2); - input.query = query.data_ptr(); - input.query_num_tokens_stride = query.stride(0); - input.query_num_heads_stride = query.stride(1); - input.cache_num_blocks_stride = key_cache.stride(0); - input.cache_num_kv_heads_stride = key_cache.stride(1); - input.blt_num_tokens_stride = block_table.stride(0); - input.key_cache = key_cache.data_ptr(); - input.value_cache = value_cache.data_ptr(); - input.output = output.data_ptr(); - input.query_start_loc = query_start_loc.data_ptr(); - input.seq_lens = seq_lens.data_ptr(); - input.block_table = block_table.data_ptr(); - input.alibi_slopes = - alibi_slopes.has_value() ? alibi_slopes->data_ptr() : nullptr; - // Attention sinks may be bf16 (native path) or fp32. Anything that is not - // bf16 must be provided as fp32 and is executed in full float precision. - if (s_aux.has_value()) { - TORCH_CHECK(s_aux->scalar_type() == at::ScalarType::BFloat16 || - s_aux->scalar_type() == at::ScalarType::Float, - "cpu_attention_with_kv_cache: s_aux (attention sinks) dtype ", - s_aux->scalar_type(), " must be bfloat16 or float32"); - input.s_aux = s_aux->data_ptr(); - input.s_aux_is_bf16 = s_aux->scalar_type() == at::ScalarType::BFloat16; - } else { - input.s_aux = nullptr; - input.s_aux_is_bf16 = false; - } - input.dynamic_causal = dynamic_causal.has_value() - ? dynamic_causal->data_ptr() - : nullptr; - input.scale = scale; - input.causal = causal; - input.sliding_window_size = sliding_window; - input.softcap = static_cast(softcap); - - if (is_fp8) { - input.k_scale_fp8 = static_cast(k_scale); - input.v_scale_fp8 = static_cast(v_scale); - TORCH_CHECK(input.metadata->isa == cpu_attention::ISA::AMX || - input.metadata->isa == cpu_attention::ISA::AMX_FP8 || - input.metadata->isa == cpu_attention::ISA::VEC, - "FP8 KV cache is only supported on x86 (AMX_FP8/AMX/VEC) ISA"); - } - - VLLM_DISPATCH_FLOATING_TYPES( - query.scalar_type(), "cpu_attention_with_kv_cache", [&]() { - CPU_ATTN_DISPATCH( - query.size(2), input.metadata->isa, kv_cache_idx, [&]() { - TORCH_CHECK_EQ(input.block_size % attn_impl::BlockSizeAlignment, - 0); - cpu_attention::AttentionMainLoop mainloop; - mainloop(&input); - }); - }); -} diff --git a/csrc/cpu/cpu_attn_amx.hpp b/csrc/cpu/cpu_attn_amx.hpp deleted file mode 100644 index 6a0341085d..0000000000 --- a/csrc/cpu/cpu_attn_amx.hpp +++ /dev/null @@ -1,636 +0,0 @@ -#ifndef CPU_ATTN_AMX_HPP -#define CPU_ATTN_AMX_HPP - -#include "cpu_attn_fp8.hpp" -#include "cpu_attn_impl.hpp" - -namespace cpu_attention { -namespace { -// AMX specific -constexpr static int64_t AMX_TILE_ROW_BYTES = 64; -constexpr static int64_t AMX_TILE_ROW_NUM = 16; -constexpr static int64_t AMX_TILE_BYTES = AMX_TILE_ROW_BYTES * AMX_TILE_ROW_NUM; - -typedef struct __tile_config { - uint8_t palette_id = 1; - uint8_t start_row = 0; - uint8_t reserved_0[14] = {0}; - uint16_t colsb[16] = {0}; - uint8_t rows[16] = {0}; -} __tilecfg; - -// 2-2-4 pattern, for 16 < m <= 32 -// TILE 0, 1: load A matrix, row num should be 16, m - 16 -// TILE 2, 3: load B matrix, row num should be 16 -// TILE 4, 5, 6, 7: store results C matrix, row num should be 16, 16, -// m - 16, m - 16 -// q_buffer_t: A (Q/P) tile type; kv_cache_t: B (K/V cache) tile type. -template -class TileGemm224 { - public: - template - FORCE_INLINE static void gemm(const int32_t m_size, void* __restrict__ a_tile, - void* __restrict__ b_tile, - float* __restrict__ c_tile, const int64_t lda, - const int64_t ldb, const int64_t ldc, - const int32_t block_size, - const int32_t dynamic_k_size, - const bool accum_c) { - TORCH_CHECK(false, "Unsupported kv cache type for TileGemm224"); - } - - FORCE_INLINE static void init_tile_config(int32_t m, __tilecfg& config) { - TORCH_CHECK(false, "Unsupported kv cache type for TileGemm224"); - } -}; - -// Dequantize one FP8 tile (AMX_TILE_ROW_NUM rows x 32 cols) to BF16. -template -FORCE_INLINE void deq_tile_amx(const uint8_t* src, c10::BFloat16* dst) { - for (int r = 0; r < AMX_TILE_ROW_NUM; ++r) { - if constexpr (std::is_same_v) { - vec_op::BF16Vec32(src + r * 32, vec_op::fp8_bf16_e4m3_tag{}) - .save(dst + r * 32); - } else { - vec_op::BF16Vec32(src + r * 32, vec_op::fp8_bf16_e5m2_tag{}) - .save(dst + r * 32); - } - } -} - -// For FP8: dequant src into scratch and return scratch. -// For BF16: return src directly (scratch is unused; the compiler elides it). -template -FORCE_INLINE const c10::BFloat16* prepare_b_tile(const kv_cache_t* src, - c10::BFloat16* scratch) { - if constexpr (std::is_same_v || - std::is_same_v) { - deq_tile_amx(reinterpret_cast(src), scratch); - return scratch; - } else { - return reinterpret_cast(src); - } -} - -// Handles both BF16 and FP8 KV cache (2-2-4 pattern). -template -class TileGemm224 { - static_assert(std::is_same_v || - std::is_same_v || - std::is_same_v, - "kv_cache_t must be BFloat16, Float8_e4m3fn, or Float8_e5m2"); - - static constexpr bool fp8_kv = - std::is_same_v || - std::is_same_v; - - static constexpr int64_t tile_elems = AMX_TILE_BYTES / sizeof(c10::BFloat16); - // BF16 path: scratch_elems=1 so the scratch array is eliminated by the - // compiler. - static constexpr int64_t scratch_elems = fp8_kv ? tile_elems : 1; - - public: - template - FORCE_INLINE static void gemm(const int32_t m_size, - c10::BFloat16* __restrict__ a_tile, - kv_cache_t* __restrict__ b_tile, - float* __restrict__ c_tile, const int64_t lda, - const int64_t ldb, const int64_t ldc, - const int32_t block_size, - const int32_t dynamic_k_size, - const bool accum_c) { - const int32_t k_times = - dynamic_k_size / (AMX_TILE_ROW_NUM * 4 / sizeof(c10::BFloat16)); - - c10::BFloat16* __restrict__ a_tile_0 = a_tile; - c10::BFloat16* __restrict__ a_tile_1 = a_tile + lda * AMX_TILE_ROW_NUM; - const int64_t a_tile_stride = [&]() { - if constexpr (phase == AttentionGemmPhase::QK) { - // q_buffer is prepacked - return AMX_TILE_ROW_BYTES; - } else if constexpr (phase == AttentionGemmPhase::PV) { - // logits_buffer is row-major - return lda * sizeof(c10::BFloat16); - } else { - TORCH_CHECK(false, "Unreachable"); - } - }(); - - kv_cache_t* __restrict__ b_tile_2 = b_tile; - kv_cache_t* __restrict__ b_tile_3 = [&]() { - if constexpr (phase == AttentionGemmPhase::QK) { - // k_cache is prepacked - return b_tile + (k_size * AMX_TILE_ROW_BYTES / 4); - } else if constexpr (phase == AttentionGemmPhase::PV) { - // v_cache is prepacked - return b_tile + (block_size * AMX_TILE_ROW_BYTES / 4); - } else { - TORCH_CHECK(false, "Unreachable"); - } - }(); - // k_cache, v_cache are prepacked - const int32_t b_tile_stride = AMX_TILE_ROW_BYTES; - - // logits_buffer, output_buffer are not prepacked - float* __restrict__ c_tile_4 = c_tile; - float* __restrict__ c_tile_5 = - c_tile_4 + AMX_TILE_ROW_BYTES / sizeof(float); - float* __restrict__ c_tile_6 = c_tile + AMX_TILE_ROW_NUM * ldc; - float* __restrict__ c_tile_7 = - c_tile_6 + AMX_TILE_ROW_BYTES / sizeof(float); - const int32_t c_tile_stride = ldc * sizeof(float); - - if (accum_c) { - _tile_loadd(4, c_tile_4, c_tile_stride); - _tile_loadd(5, c_tile_5, c_tile_stride); - _tile_loadd(6, c_tile_6, c_tile_stride); - _tile_loadd(7, c_tile_7, c_tile_stride); - } else { - _tile_zero(4); - _tile_zero(5); - _tile_zero(6); - _tile_zero(7); - } - - alignas(64) c10::BFloat16 scratch_2[scratch_elems]; - alignas(64) c10::BFloat16 scratch_3[scratch_elems]; - for (int32_t k = 0; k < k_times; ++k) { - const c10::BFloat16* load_2 = prepare_b_tile(b_tile_2, scratch_2); - const c10::BFloat16* load_3 = prepare_b_tile(b_tile_3, scratch_3); - - _tile_loadd(0, a_tile_0, a_tile_stride); - _tile_stream_loadd(2, const_cast(load_2), b_tile_stride); - _tile_dpbf16ps(4, 0, 2); - _tile_stream_loadd(3, const_cast(load_3), b_tile_stride); - _tile_dpbf16ps(5, 0, 3); - _tile_loadd(1, a_tile_1, a_tile_stride); - _tile_dpbf16ps(6, 1, 2); - _tile_dpbf16ps(7, 1, 3); - - // update ptrs - if constexpr (phase == AttentionGemmPhase::QK) { - // Q buffer is prepacked - a_tile_0 += AMX_TILE_BYTES / sizeof(c10::BFloat16); - a_tile_1 += AMX_TILE_BYTES / sizeof(c10::BFloat16); - } else if constexpr (phase == AttentionGemmPhase::PV) { - // P buffer is not prepacked - a_tile_0 += AMX_TILE_ROW_BYTES / sizeof(c10::BFloat16); - a_tile_1 += AMX_TILE_ROW_BYTES / sizeof(c10::BFloat16); - } else { - TORCH_CHECK(false, "Unreachable"); - } - b_tile_2 += AMX_TILE_BYTES / sizeof(c10::BFloat16); - b_tile_3 += AMX_TILE_BYTES / sizeof(c10::BFloat16); - } - - _tile_stored(4, c_tile_4, c_tile_stride); - _tile_stored(5, c_tile_5, c_tile_stride); - _tile_stored(6, c_tile_6, c_tile_stride); - _tile_stored(7, c_tile_7, c_tile_stride); - } - - FORCE_INLINE static void init_tile_config(int32_t m, __tilecfg& config) { - const int32_t m_0 = AMX_TILE_ROW_NUM; - const int32_t m_1 = m - AMX_TILE_ROW_NUM; - config.rows[0] = m_0; - config.rows[1] = m_1; - config.rows[2] = AMX_TILE_ROW_NUM; - config.rows[3] = AMX_TILE_ROW_NUM; - config.rows[4] = m_0; - config.rows[5] = m_0; - config.rows[6] = m_1; - config.rows[7] = m_1; - _tile_loadconfig(&config); - } -}; - -// 1-2-2 pattern, for 0 < m <= 16 -// TILE 0, (1): load A matrix, use extra 1 tile for prefetch, row num should -// be m, m -// TILE 2, 3, (4, 5): load B matrix, use extra 2 tiles for prefetch, row num -// should be 16 -// TILE 6, 7: store results C matrix, row num should be m -// q_buffer_t: A (Q/P) tile type; kv_cache_t: B (K/V cache) tile type. -template -class TileGemm122 { - public: - template - FORCE_INLINE static void gemm(const int32_t m_size, void* __restrict__ a_tile, - void* __restrict__ b_tile, - float* __restrict__ c_tile, const int64_t lda, - const int64_t ldb, const int64_t ldc, - const int32_t block_size, - const int32_t dynamic_k_size, - const bool accum_c) { - TORCH_CHECK(false, "Unsupported kv cache type for TileGemm122"); - } - - FORCE_INLINE static void init_tile_config(int32_t m, __tilecfg& config) { - TORCH_CHECK(false, "Unsupported kv cache type for TileGemm122"); - } -}; - -// Handles both BF16 and FP8 KV cache (1-2-2 pattern). -template -class TileGemm122 { - static_assert(std::is_same_v || - std::is_same_v || - std::is_same_v, - "kv_cache_t must be BFloat16, Float8_e4m3fn, or Float8_e5m2"); - - static constexpr bool fp8_kv = - std::is_same_v || - std::is_same_v; - - static constexpr int64_t tile_elems = AMX_TILE_BYTES / sizeof(c10::BFloat16); - static constexpr int64_t scratch_elems = fp8_kv ? tile_elems : 1; - - public: - template - FORCE_INLINE static void gemm(const int32_t m_size, - c10::BFloat16* __restrict__ a_tile, - kv_cache_t* __restrict__ b_tile, - float* __restrict__ c_tile, const int64_t lda, - const int64_t ldb, const int64_t ldc, - const int32_t block_size, - const int32_t dynamic_k_size, - const bool accum_c) { - c10::BFloat16* __restrict__ a_tile_0 = a_tile; - c10::BFloat16* __restrict__ a_tile_1 = [&]() { - if constexpr (phase == AttentionGemmPhase::QK) { - // q_buffer is prepacked - return a_tile + AMX_TILE_BYTES / sizeof(c10::BFloat16); - } else if constexpr (phase == AttentionGemmPhase::PV) { - // logits_buffer is row-major - return a_tile + AMX_TILE_ROW_BYTES / sizeof(c10::BFloat16); - } else { - TORCH_CHECK(false, "Unreachable"); - } - }(); - const int64_t a_tile_stride = [&]() { - if constexpr (phase == AttentionGemmPhase::QK) { - // q_buffer is prepacked - return AMX_TILE_ROW_BYTES; - } else if constexpr (phase == AttentionGemmPhase::PV) { - // logits_buffer is row-major - return lda * sizeof(c10::BFloat16); - } else { - TORCH_CHECK(false, "Unreachable"); - } - }(); - - kv_cache_t* __restrict__ b_tile_2 = b_tile; - kv_cache_t* __restrict__ b_tile_3 = [&]() { - if constexpr (phase == AttentionGemmPhase::QK) { - return b_tile + (k_size * AMX_TILE_ROW_BYTES / 4); - } else if constexpr (phase == AttentionGemmPhase::PV) { - return b_tile + (block_size * AMX_TILE_ROW_BYTES / 4); - } else { - TORCH_CHECK(false, "Unreachable"); - } - }(); - kv_cache_t* __restrict__ b_tile_4 = - b_tile_2 + AMX_TILE_BYTES / sizeof(c10::BFloat16); - kv_cache_t* __restrict__ b_tile_5 = - b_tile_3 + AMX_TILE_BYTES / sizeof(c10::BFloat16); - int64_t b_stride = AMX_TILE_ROW_BYTES; - - float* __restrict__ c_tile_6 = c_tile; - float* __restrict__ c_tile_7 = c_tile + AMX_TILE_ROW_BYTES / sizeof(float); - int64_t c_stride = ldc * sizeof(float); - - const int32_t k_times = - dynamic_k_size / (AMX_TILE_ROW_NUM * 4 / sizeof(c10::BFloat16)); - const int32_t k_group_times = k_times / 2; - const bool has_tail = (k_times % 2 == 1); - - if (accum_c) { - _tile_loadd(6, c_tile_6, c_stride); - _tile_loadd(7, c_tile_7, c_stride); - } else { - _tile_zero(6); - _tile_zero(7); - } - - alignas(64) c10::BFloat16 scratch_2[scratch_elems]; - alignas(64) c10::BFloat16 scratch_3[scratch_elems]; - alignas(64) c10::BFloat16 scratch_4[scratch_elems]; - alignas(64) c10::BFloat16 scratch_5[scratch_elems]; - for (int32_t k = 0; k < k_group_times; ++k) { - const c10::BFloat16* load_2 = prepare_b_tile(b_tile_2, scratch_2); - const c10::BFloat16* load_3 = prepare_b_tile(b_tile_3, scratch_3); - const c10::BFloat16* load_4 = prepare_b_tile(b_tile_4, scratch_4); - const c10::BFloat16* load_5 = prepare_b_tile(b_tile_5, scratch_5); - - _tile_loadd(0, a_tile_0, a_tile_stride); - _tile_stream_loadd(2, const_cast(load_2), b_stride); - _tile_dpbf16ps(6, 0, 2); - _tile_stream_loadd(3, const_cast(load_3), b_stride); - _tile_dpbf16ps(7, 0, 3); - _tile_loadd(1, a_tile_1, a_tile_stride); - _tile_stream_loadd(4, const_cast(load_4), b_stride); - _tile_dpbf16ps(6, 1, 4); - _tile_stream_loadd(5, const_cast(load_5), b_stride); - _tile_dpbf16ps(7, 1, 5); - - // update ptrs - if constexpr (phase == AttentionGemmPhase::QK) { - // Q buffer is prepacked - a_tile_0 += 2 * AMX_TILE_BYTES / sizeof(c10::BFloat16); - a_tile_1 += 2 * AMX_TILE_BYTES / sizeof(c10::BFloat16); - } else if constexpr (phase == AttentionGemmPhase::PV) { - // P buffer is not prepacked - a_tile_0 += 2 * AMX_TILE_ROW_BYTES / sizeof(c10::BFloat16); - a_tile_1 += 2 * AMX_TILE_ROW_BYTES / sizeof(c10::BFloat16); - } - b_tile_2 += 2 * AMX_TILE_BYTES / sizeof(c10::BFloat16); - b_tile_3 += 2 * AMX_TILE_BYTES / sizeof(c10::BFloat16); - b_tile_4 += 2 * AMX_TILE_BYTES / sizeof(c10::BFloat16); - b_tile_5 += 2 * AMX_TILE_BYTES / sizeof(c10::BFloat16); - } - - if (has_tail) { - const c10::BFloat16* load_2 = prepare_b_tile(b_tile_2, scratch_2); - const c10::BFloat16* load_3 = prepare_b_tile(b_tile_3, scratch_3); - - _tile_loadd(0, a_tile_0, a_tile_stride); - _tile_stream_loadd(2, const_cast(load_2), b_stride); - _tile_dpbf16ps(6, 0, 2); - _tile_stream_loadd(3, const_cast(load_3), b_stride); - _tile_dpbf16ps(7, 0, 3); - } - - _tile_stored(6, c_tile_6, c_stride); - _tile_stored(7, c_tile_7, c_stride); - } - - FORCE_INLINE static void init_tile_config(int32_t m, __tilecfg& config) { - config.rows[0] = m; - config.rows[1] = m; - config.rows[2] = AMX_TILE_ROW_NUM; - config.rows[3] = AMX_TILE_ROW_NUM; - config.rows[4] = AMX_TILE_ROW_NUM; - config.rows[5] = AMX_TILE_ROW_NUM; - config.rows[6] = m; - config.rows[7] = m; - _tile_loadconfig(&config); - } -}; - -} // namespace - -template -class AttentionImpl { - static constexpr bool fp8_kv = - std::is_same_v || - std::is_same_v; - - public: - using query_t = scalar_t; - using q_buffer_t = scalar_t; - using kv_cache_t = kv_cache_scalar_t; - using logits_buffer_t = float; - using partial_output_buffer_t = float; - using prob_buffer_t = scalar_t; - - constexpr static int64_t BlockSizeAlignment = - 32; // AMX_TILE_ROW_NUM = 16 tokens/tile; 32 = 2 tiles - constexpr static int64_t HeadDimAlignment = - 2 * (AMX_TILE_ROW_BYTES / 4); // headdim num unit of PV phase - constexpr static int64_t MaxQHeadNumPerIteration = 32; - constexpr static int64_t HeadDim = head_dim; - constexpr static ISA ISAType = ISA::AMX; - constexpr static bool scale_on_logits = true; - - float k_scale = 1.0f; - float v_scale = 1.0f; - - public: - AttentionImpl() : current_q_head_num_(0) { - // Use all columns in AMX tiles - vec_op::unroll_loop([&](int i) { amx_tile_config_.colsb[i] = 64; }); - } - - ~AttentionImpl() { _tile_release(); } - - void init_from_input(const AttentionInput* input) { - if constexpr (fp8_kv) { - k_scale = input->k_scale_fp8; - v_scale = input->v_scale_fp8; - } - } - - float get_output_v_scale() const noexcept { - if constexpr (fp8_kv) { - // AMX dequant places FP8 payload into a BF16 field (exponent bias 127). - // Correction = 2^(127 - FP8_bias): E4M3 bias=7 → 2^120, E5M2 bias=15 → - // 2^112. - constexpr float bias = - std::is_same_v ? 0x1p112f : 0x1p120f; - return v_scale * bias; - } - return 1.0f; - } - - template