group: Kernels
depends_on: 
  - image-build
steps:
- label: ":nvidia: (H200 MIG 18GB) vLLM IR"
  key: vllm-ir-tests
  timeout_in_minutes: 35
  device: h200_18gb
  working_dir: "/vllm-workspace/"
  source_file_dependencies:
    - vllm/ir
    - vllm/kernels
  commands:
    - pytest -v -s tests/ir
    - pytest -v -s tests/kernels/ir
  mirror:
    amd:
      label: ":amd: (MI355 DPX) vLLM IR"
      dind: false
      device: mi355_dpx
      timeout_in_minutes: 50
      working_dir: "/vllm-workspace/"
      depends_on:
      - image-build-amd
      source_file_dependencies:
      - tests/ir/
      - tests/kernels/ir/
      - vllm/ir
      - vllm/kernels
      - vllm/_aiter_ops.py
      - vllm/platforms/rocm.py

- label: ":nvidia: (H200 MIG 35GB) Core Operation Kernels Shard %N"
  device: h200_35gb
  key: kernels-core-operation-test
  timeout_in_minutes: 120
  source_file_dependencies:
  - csrc/
  - tests/kernels/core
  - tests/kernels/test_concat_mla_q.py
  - tests/kernels/test_fused_qk_norm_rope_gate.py
  commands:
    - pytest -v -s kernels/core --ignore=kernels/core/test_minimax_reduce_rms.py kernels/test_concat_mla_q.py kernels/test_fused_qk_norm_rope_gate.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
  parallelism: 3
  mirror:
    amd:
      label: ":amd: (MI300) Core Operation Kernels Shard %N"
      dind: false
      device: mi300_1
      timeout_in_minutes: 45
      working_dir: "/vllm-workspace/tests"
      depends_on:
      - image-build-amd
      source_file_dependencies:
      - csrc/
      - tests/kernels/core
      - tests/kernels/test_top_k_per_row.py
      - tests/kernels/test_concat_mla_q.py
      - tests/kernels/test_rocm_fp32_router_gemm.py
      - vllm/model_executor/layers/fused_moe/router/gate_linear.py
      - vllm/model_executor/layers/fused_moe/router/rocm_fp32_router_gemm.py
      - vllm/model_executor/layers/rotary_embedding/
      - vllm/_aiter_ops.py
      - vllm/platforms/rocm.py
      - tests/kernels/test_fused_qk_norm_rope_gate.py
      commands:
      - pytest -v -s kernels/core --ignore=kernels/core/test_minimax_reduce_rms.py kernels/test_fused_qk_norm_rope_gate.py kernels/test_concat_mla_q.py kernels/test_top_k_per_row.py kernels/test_rocm_fp32_router_gemm.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT

- label: ":nvidia: (H100) MiniMax Reduce RMS Kernels"
  key: kernels-minimax-reduce-rms-test-2-gpus
  timeout_in_minutes: 20
  num_devices: 2
  device: h100
  source_file_dependencies:
  - csrc/minimax_reduce_rms_kernel.cu
  - csrc/minimax_reduce_rms_kernel.h
  - csrc/ops.h
  - csrc/torch_bindings.cpp
  - vllm/model_executor/layers/minimax_rms_norm/
  - vllm/model_executor/layers/mamba/linear/minimax_linear_attn.py
  - tests/kernels/core/test_minimax_reduce_rms.py
  commands:
    - pytest -v -s kernels/core/test_minimax_reduce_rms.py
  mirror:
    amd:
      label: ":amd: (MI355) MiniMax Reduce RMS Kernels"
      dind: false
      device: mi355_2
      timeout_in_minutes: 45
      working_dir: "/vllm-workspace/tests"
      depends_on:
      - image-build-amd
      source_file_dependencies:
      - vllm/model_executor/layers/minimax_rms_norm/
      - vllm/distributed/
      - vllm/_aiter_ops.py
      - vllm/envs.py
      - vllm/platforms/rocm.py
      - tests/kernels/core/test_minimax_reduce_rms.py
      - tests/utils.py

- label: ":nvidia: (H100) DeepSeek V4 Kernels"
  key: deepseek-v4-kernel-test-h100
  timeout_in_minutes: 30
  device: h100
  source_file_dependencies:
  - csrc/fused_deepseek_v4_qnorm_rope_kv_insert_kernel.cu
  - vllm/models/deepseek_v4/common/ops/
  - tests/kernels/test_fused_deepseek_v4_qnorm_rope_kv_insert.py
  - tests/kernels/test_top_k_per_row.py # it runs on Blackwell too - some kernels have arch-specific optimizations
  commands:
    - pytest -v -s kernels/test_fused_deepseek_v4_*.py
    - pytest -v -s kernels/test_top_k_per_row.py
  mirror:
    amd:
      label: ":amd: (MI355 DPX) DeepSeek V4 Fused Kernels"
      dind: false
      device: mi355_dpx
      num_devices: 1
      timeout_in_minutes: 60
      working_dir: /vllm-workspace/tests
      depends_on:
      - image-build-amd
      source_file_dependencies:
      - csrc/fused_deepseek_v4_qnorm_rope_kv_insert_kernel.cu
      - vllm/models/deepseek_v4/common/ops/
      - tests/kernels/test_fused_deepseek_v4_qnorm_rope_kv_insert.py
      - vllm/platforms/rocm.py
      commands:
      - pytest -v -s kernels/test_fused_deepseek_v4_qnorm_rope_kv_insert.py

- label: ":nvidia: (B200) DeepSeek V4 Kernels"
  key: deepseek-v4-kernel-test-b200
  timeout_in_minutes: 20
  device: b200-k8s
  source_file_dependencies:
  - csrc/fused_deepseek_v4_qnorm_rope_kv_insert_kernel.cu
  - vllm/models/deepseek_v4/common/ops/
  - vllm/models/deepseek_v4/nvidia/
  - vllm/utils/deep_gemm.py
  - tests/kernels/test_fused_deepseek_v4_qnorm_rope_kv_insert.py
  - tests/models/test_deepseek_v4_mega_moe.py
  commands:
    - pytest -v -s kernels/test_fused_deepseek_v4_*.py
    - pytest -v -s models/test_deepseek_v4_mega_moe.py
  mirror:
    amd:
      label: ":amd: (MI355) DeepSeek V4 MoE Integration"
      dind: false
      device: mi355_1
      num_devices: 1
      timeout_in_minutes: 20
      working_dir: /vllm-workspace/tests
      depends_on:
      - image-build-amd
      source_file_dependencies:
      - tests/models/test_deepseek_v4_vl_rocm.py
      - tests/models/test_deepseek_v4_aiter_mega_moe.py
      - vllm/models/deepseek_v4/amd/
      - vllm/models/deepseek_v4/common/
      - vllm/model_executor/layers/fused_moe/
      - vllm/model_executor/layers/activation.py
      - vllm/model_executor/layers/linear.py
      - vllm/config/
      - vllm/forward_context.py
      - vllm/v1/worker/workspace.py
      - vllm/_aiter_ops.py
      - vllm/platforms/rocm.py
      - vllm/envs.py
      commands:
      # Common qnorm/RoPE/cache kernels already run in the H100 AMD mirror.
      # Pin the tested BF16 Triton routing and shared-expert path before imports.
      - VLLM_ROCM_USE_AITER_MOE=0 pytest -v -s models/test_deepseek_v4_vl_rocm.py
      - pytest -v -s models/test_deepseek_v4_aiter_mega_moe.py

# Own job so mHC never shares a process with FlashInfer's import-time
# ptxas override.
- label: ":nvidia: (B200) mHC Kernels"
  key: kernels-mhc-test-b200
  timeout_in_minutes: 25
  device: b200-k8s
  source_file_dependencies:
  - csrc/
  - vllm/
  - "!vllm/distributed/kv_transfer/"
  - tests/kernels/
  commands:
    - pytest -v -s kernels/mhc
  mirror:
    amd:
      label: ":amd: (MI355) mHC Kernels"
      dind: false
      device: mi355_1
      timeout_in_minutes: 50
      working_dir: "/vllm-workspace/tests"
      depends_on:
      - image-build-amd
      source_file_dependencies:
      - csrc/
      - vllm/
      - tests/kernels/
      - vllm/platforms/rocm.py

- label: ":nvidia: (B200) FLA Ops Kernels"
  key: kernels-fla-ops-test-b200
  timeout_in_minutes: 25
  device: b200-k8s
  source_file_dependencies:
  - csrc/
  - vllm/
  - "!vllm/distributed/kv_transfer/"
  - tests/kernels/
  commands:
    - pytest -v -s kernels/fla
  mirror:
    amd:
      label: ":amd: (MI355) FLA Ops Kernels"
      dind: false
      device: mi355_1
      timeout_in_minutes: 50
      working_dir: "/vllm-workspace/tests"
      depends_on:
      - image-build-amd
      source_file_dependencies:
      - csrc/
      - vllm/
      - tests/kernels/
      - vllm/platforms/rocm.py

# Catch-all for test files at the tests/kernels root. This job collects
# the whole root so new files are wired by default.
# Files with dedicated jobs elsewhere in this file are excluded via --ignore
# (test_bf16x3_router_gemm_cutedsl and test_ll_bf16_gemm run in their own jobs /
# Kernels (B200)).
- label: ":nvidia: (B200) Miscellaneous Kernels"
  key: kernels-root-misc-test-b200
  timeout_in_minutes: 30
  device: b200-k8s
  source_file_dependencies:
  - csrc/
  - vllm/
  - "!vllm/distributed/kv_transfer/"
  - tests/kernels/
  commands:
    - pytest -v -s kernels/
        --ignore=kernels/attention
        --ignore=kernels/core
        --ignore=kernels/fla
        --ignore=kernels/helion
        --ignore=kernels/ir
        --ignore=kernels/mamba
        --ignore=kernels/mhc
        --ignore=kernels/moe
        --ignore=kernels/quantization
        --ignore=kernels/test_concat_mla_q.py
        --ignore=kernels/test_fused_qk_norm_rope_gate.py
        --ignore=kernels/test_fused_deepseek_v4_qnorm_rope_kv_insert.py
        --ignore=kernels/test_top_k_per_row.py
        --ignore=kernels/test_cpu_fused_sampling.py
        --ignore=kernels/test_bf16x3_router_gemm_cutedsl.py
        --ignore=kernels/test_bf16x3_router_gemm_rocm.py
        --ignore=kernels/test_ll_bf16_gemm.py
        --ignore=kernels/test_shuffle_rows.py
    # BROKEN on main, pending kernel fixes (B200):
    #   test_shuffle_rows.py (1: test_shuffle_rows_edge_cases)
  mirror:
    amd:
      label: ":amd: (MI355) Miscellaneous Kernels"
      dind: false
      device: mi355_1
      timeout_in_minutes: 55
      working_dir: "/vllm-workspace/tests"
      depends_on:
      - image-build-amd
      source_file_dependencies:
      - csrc/
      - vllm/
      - tests/kernels/
      - vllm/platforms/rocm.py
      # Keep the ROCm exclusions from #50519: incompatible FlyDSL TurboQuant
      # and a CUDA source check whose files are absent from the AMD test image.
      commands:
      - pytest -v -s
          kernels/test_apply_repetition_penalties.py
          kernels/test_awq_int4_to_int8.py
          kernels/test_bf16_skinny_gemm.py
          kernels/test_bf16x3_router_gemm_rocm.py
          kernels/test_cache_kernels.py
          kernels/test_compressor_kv_cache.py
          kernels/test_cp_gather_fp8.py
          kernels/test_deepseek_v4_cpu_kernels.py
          kernels/test_dsv41_mega_attn_layouts.py
          kernels/test_engram.py
          kernels/test_flashinfer_bf16_gemm.py
          kernels/test_flex_attention.py
          kernels/test_fp32_router_gemm.py
          kernels/test_fused_deepseek_v32_norm_rope.py
          kernels/test_fused_indexer_q_rope_quant.py
          kernels/test_fused_inv_rope_fp8_quant.py
          kernels/test_fused_minimax_m3_qknorm_rope_kv_insert.py
          kernels/test_fused_quant_activation.py
          kernels/test_gate_linear_rocm_dispatch.py
          kernels/test_gemm_rs_ar.py
          kernels/test_kpool_decode_update_batched.py
          kernels/test_minimax_m3_amd_ops.py
          kernels/test_minimax_m3_sparse_attn_fp8_scale.py
          kernels/test_onednn.py
          kernels/test_relu2_fp8_quant.py
          kernels/test_rocm_fp32_router_gemm.py
          --deselect=tests/kernels/test_bf16_skinny_gemm.py::test_every_dsv3_routed_shape_is_instantiated

- label: ":nvidia: (L4) Attention Kernels Shard %N"
  key: kernels-attention-test
  device: l4
  num_devices: 1
  timeout_in_minutes: 30
  source_file_dependencies:
  - csrc/attention/
  - csrc/core/vec_dtypes/
  - vllm/v1/attention
    # TODO: remove this dependency (https://github.com/vllm-project/vllm/issues/32267)
  - vllm/model_executor/layers/attention
  - vllm/utils/flashinfer.py
  - tests/kernels/attention
  commands:
    - pytest -v -s kernels/attention --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
  parallelism: 7
  mirror:
    amd:
      label: ":amd: (MI300) Attention Kernels Shard %N"
      dind: false
      device: mi300_1
      timeout_in_minutes: 90
      parallelism: 6
      depends_on:
      - image-build-amd
      source_file_dependencies:
      - csrc/attention/
      - csrc/core/vec_dtypes/
      - vllm/v1/attention
      - vllm/model_executor/layers/attention
      - tests/kernels/attention
      - vllm/_aiter_ops.py
      - vllm/envs.py
      - vllm/platforms/rocm.py
      commands:
      - pytest -v -s kernels/attention --ignore=kernels/attention/test_triton_unified_attention_diffkv.py --ignore=kernels/attention/test_rocm_aiter_mla_decode.py --ignore=kernels/attention/test_rocm_aiter_mla_decode_metadata.py --ignore=kernels/attention/test_rocm_aiter_mla_fp8_support.py --ignore=kernels/attention/test_rocm_aiter_mla_op_registration.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT

- label: ":nvidia: (H100) Attention DiffKV Kernels"
  key: kernels-attention-diffkv-test-h100
  timeout_in_minutes: 20
  device: h100
  num_devices: 1
  source_file_dependencies:
  - vllm/v1/attention/ops/triton_unified_attention_diffkv.py
  - vllm/v1/attention/backends/triton_attn_diffkv.py
  - vllm/v1/attention/backends/flash_attn_diffkv.py
  - tests/kernels/attention/test_triton_unified_attention_diffkv.py
  commands:
    - pytest -v -s kernels/attention/test_triton_unified_attention_diffkv.py
  mirror:
    amd:
      label: ":amd: (MI355 DPX) Attention DiffKV Kernels"
      dind: false
      device: mi355_dpx
      num_devices: 1
      timeout_in_minutes: 35
      working_dir: /vllm-workspace/tests
      depends_on:
      - image-build-amd
      source_file_dependencies:
      - vllm/v1/attention/ops/triton_unified_attention_diffkv.py
      - vllm/v1/attention/backends/triton_attn_diffkv.py
      - tests/kernels/attention/test_triton_unified_attention.py
      - tests/kernels/attention/test_triton_unified_attention_diffkv.py
      - vllm/platforms/rocm.py
      commands:
      - pytest -v -s kernels/attention/test_triton_unified_attention_diffkv.py

- label: ":nvidia: (H100) MLA Kernel Test"
  key: kernels-flashmla-test-h100
  timeout_in_minutes: 25
  device: h100
  num_devices: 1
  source_file_dependencies:
  - cmake/external_projects/flashmla.cmake
  - vllm/v1/attention/ops/flashmla.py
  - vllm/v1/attention/backends/mla/flashmla.py
  - vllm/v1/attention/backends/mla/flashmla_sparse.py
  - vllm/v1/attention/backends/mla/flashinfer_mla_sparse_sm90.py
  - vllm/utils/flashinfer.py
  - tests/kernels/attention/test_flashinfer_mla_decode.py
  - tests/kernels/attention/test_flashmla.py
  - tests/kernels/attention/test_flashmla_sparse.py
  - tests/kernels/attention/test_mla_cross_layer_kernel_equivalence.py
  commands:
    - pytest -v -s kernels/attention/test_flashmla.py
    - pytest -v -s kernels/attention/test_flashmla_sparse.py
    - pytest -v -s kernels/attention/test_flashinfer_mla_decode.py
    - pytest -v -s kernels/attention/test_mla_cross_layer_kernel_equivalence.py
  mirror:
    amd:
      label: ":amd: (MI355 DPX) MLA Kernels"
      dind: false
      device: mi355_dpx
      num_devices: 1
      timeout_in_minutes: 40
      working_dir: /vllm-workspace/
      depends_on:
      - image-build-amd
      source_file_dependencies:
      - vllm/v1/attention/backends/mla/rocm_aiter_mla.py
      - vllm/v1/attention/selector.py
      - vllm/platforms/rocm.py
      - vllm/_aiter_ops.py
      - tests/kernels/attention/test_rocm_aiter_mla_decode.py
      - tests/kernels/attention/test_rocm_aiter_mla_decode_metadata.py
      - tests/kernels/attention/test_rocm_aiter_mla_fp8_support.py
      - tests/kernels/attention/test_rocm_aiter_mla_op_registration.py
      commands:
      - amd-smi
      - pytest -v -s tests/kernels/attention/test_rocm_aiter_mla_decode_metadata.py
      - pytest -v -s tests/kernels/attention/test_rocm_aiter_mla_decode.py
      - pytest -v -s tests/kernels/attention/test_rocm_aiter_mla_fp8_support.py
      - pytest -v -s tests/kernels/attention/test_rocm_aiter_mla_op_registration.py

- label: ":nvidia: (L4) Quantization Kernels Shard %N"
  key: kernels-quantization-test
  device: l4
  num_devices: 1
  timeout_in_minutes: 30
  source_file_dependencies:
  - csrc/quantization/
  - vllm/model_executor/layers/quantization
  - tests/kernels/quantization
  commands:
    - pytest -v -s kernels/quantization --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
  parallelism: 6
  mirror:
    amd:
      label: ":amd: (MI300) Quantization Kernels Shard %N"
      dind: false
      device: mi300_1
      timeout_in_minutes: 120
      source_file_dependencies:
      - csrc/quantization/
      - vllm/model_executor/layers/quantization
      - vllm/config/
      - tests/kernels/quantization
      - tests/kernels/quantization/test_rocm_skinny_gemms.py
      - vllm/_aiter_ops.py
      - vllm/kernels/aiter_ops.py
      - vllm/platforms/rocm.py
      - vllm/model_executor/kernels/
      - csrc/rocm/
      - vllm/v1/attention/backends/rocm_aiter_fa.py
      - tests/kernels/quant_utils.py
      - tests/kernels/utils.py
      - vllm/_custom_ops.py
      - vllm/envs.py
      depends_on:
      - image-build-amd
      commands:
      - pytest -v -s kernels/quantization -k "not test_nvfp4_dense_emulation" --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT

- label: ":nvidia: (L4) MoE Kernels Shard %N"
  key: kernels-moe-test
  device: l4
  num_devices: 1
  timeout_in_minutes: 50
  source_file_dependencies:
  - csrc/quantization/w8a8/cutlass/moe/
  - csrc/moe/
  - tests/kernels/moe
  - vllm/model_executor/layers/fused_moe/
  - vllm/distributed/device_communicators/
  - vllm/envs.py
  - vllm/config
  commands:
    - pytest -v -s kernels/moe --ignore=kernels/moe/test_modular_oai_triton_moe.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
    - pytest -v -s kernels/moe/test_modular_oai_triton_moe.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
  parallelism: 5
  mirror:
    amd:
      label: ":amd: (MI300) MoE Kernels Shard %N"
      dind: false
      device: mi300_1
      timeout_in_minutes: 55
      source_file_dependencies:
      - csrc/quantization/w8a8/cutlass/moe/
      - csrc/moe/
      - tests/kernels/moe
      - vllm/model_executor/layers/fused_moe/
      - vllm/distributed/device_communicators/
      - vllm/envs.py
      - vllm/config
      - vllm/_aiter_ops.py
      - vllm/platforms/rocm.py
      - csrc/rocm/
      - tests/kernels/utils.py
      - vllm/model_executor/layers/quantization/mxfp4.py
      - vllm/model_executor/layers/quantization/utils/quant_utils.py
      depends_on:
      - image-build-amd

- label: ":nvidia: (H200 MIG 35GB) Mamba Kernels"
  device: h200_35gb
  key: kernels-mamba-test
  automatic_shard: true
  timeout_in_minutes: 60
  source_file_dependencies:
  - csrc/mamba/
  - tests/kernels/mamba
  - vllm/model_executor/layers/mamba/gdn
  - vllm/model_executor/layers/mamba/ops
  commands:
    - pytest -v -s kernels/mamba
  mirror:
    amd:
      label: ":amd: (MI250) Mamba Kernels"
      dind: false
      device: mi250_1
      timeout_in_minutes: 55
      depends_on:
      - image-build-amd
      source_file_dependencies:
      - vllm/platforms/rocm.py
      - vllm/third_party/flash_linear_attention/

- label: ":nvidia: (H100) DeepGEMM Kernels"
  key: kernels-deepgemm-test-h100
  timeout_in_minutes: 35
  device: h100
  num_devices: 1
  source_file_dependencies:
  - cmake/external_projects/deepgemm.cmake
  - tools/install_deepgemm.sh
  - tools/build_deepgemm_C.py
  - tools/setup_deepgemm_pythons.sh
  - tools/check_wheel_deepgemm.py
  - vllm/utils/deep_gemm.py
  - vllm/model_executor/layers/fused_moe
  - vllm/model_executor/layers/quantization
  - tests/kernels/quantization/test_block_fp8.py
  - tests/kernels/moe/test_deepgemm.py
  - tests/kernels/moe/test_batched_deepgemm.py
  - tests/kernels/attention/test_deepgemm_attention.py
  - tests/quantization/test_cutlass_w4a16.py
  commands:
    - python3 ../tools/check_wheel_deepgemm.py
    - pytest -v -s kernels/quantization/test_block_fp8.py
    - pytest -v -s kernels/moe/test_deepgemm.py
    - pytest -v -s kernels/moe/test_batched_deepgemm.py
    - pytest -v -s kernels/attention/test_deepgemm_attention.py
    - pytest -v -s quantization/test_cutlass_w4a16.py

- label: ":nvidia: (B200) MiniMax-M3 Attention Kernels"
  key: kernels-minimax-m3-attention-b200
  timeout_in_minutes: 30
  working_dir: "/vllm-workspace/"
  device: b200-k8s
  num_devices: 1
  source_file_dependencies:
  - vllm/models/minimax_m3/
  - vllm/config/cache.py
  - vllm/v1/kv_cache_interface.py
  - tests/v1/attention/utils.py
  - tests/kernels/attention/test_minimax_m3.py
  commands:
    - pytest -v -s tests/kernels/attention/test_minimax_m3.py

- label: ":nvidia: (B200) Kernels Shard %N"
  key: kernels-b200
  timeout_in_minutes: 30
  parallelism: 3
  working_dir: "/vllm-workspace/"
  device: b200-k8s
  # optional: true
  source_file_dependencies:
  - csrc/quantization/fp4/
  - csrc/nvfp4_kv_cache_kernels.cu
  - csrc/quantization/w8a8/cutlass/
  - csrc/attention/mla/
  - csrc/quantization/w8a8/cutlass/moe/
  - vllm/model_executor/layers/fusion/
  - vllm/model_executor/layers/fused_moe/experts/cutlass_moe.py
  - vllm/model_executor/layers/fused_moe/experts/flashinfer_cutlass_moe.py
  - vllm/model_executor/layers/fused_moe/experts/trtllm_nvfp4_moe.py
  - vllm/model_executor/layers/fused_moe/oracle/nvfp4.py
  - vllm/model_executor/layers/fused_moe/prepare_finalize/flashinfer_nvlink_one_sided.py
  - vllm/model_executor/layers/fused_moe/prepare_finalize/flashinfer_nvlink_two_sided.py
  - vllm/model_executor/layers/quantization/utils/flashinfer_utils.py
  - vllm/v1/attention/backends/flashinfer.py
  - vllm/v1/attention/backends/mla/cutlass_mla.py
  - vllm/v1/attention/backends/mla/flashinfer_mla.py
  - vllm/v1/attention/selector.py
  - vllm/platforms/cuda.py
  - vllm/model_executor/kernels/linear/cute_dsl/ll_bf16.py
  - vllm/model_executor/kernels/linear/cute_dsl/_ll_bf16_dotprod.py
  - vllm/model_executor/kernels/linear/cute_dsl/_ll_bf16_splitk.py
  - vllm/cute_utils/
  - vllm/model_executor/layers/mamba/ops/gdn_chunk_cutedsl/
  - vllm/model_executor/layers/fused_moe/router/bf16x3_router_gemm_cutedsl.py
  - cmake/external_projects/fmha_sm100.cmake
  - vllm/models/minimax_m3/common/sparse_attention.py
  - vllm/models/minimax_m3/nvidia/
  - vllm/config/attention.py
  - vllm/v1/attention/backends/registry.py
  - vllm/_custom_ops.py
  - csrc/fused_minimax_m3_qknorm_rope_kv_insert_kernel.cu
  - csrc/ops.h
  - csrc/torch_bindings.cpp
  - tests/kernels/attention/test_flashinfer_mla_decode.py
  - tests/kernels/attention/test_minimax_m3_msa_cutlass_sparse_decode.py
  - tests/kernels/attention/test_cache.py
  - tests/kernels/quantization/nvfp4_utils.py
  - tests/kernels/test_fused_minimax_m3_qknorm_rope_kv_insert.py
  - tests/kernels/mamba/test_gdn_prefill_cutedsl.py
  - tests/kernels/test_bf16x3_router_gemm_cutedsl.py
  - tests/kernels/test_ll_bf16_gemm.py
  - tests/kernels/test_top_k_per_row.py
  - tests/kernels/quantization/test_silu_mul_nvfp4_quant.py
  - tests/kernels/quantization/test_block_fp8.py
  - tests/models/quantization/test_nvfp4.py
  commands:
    - nvidia-smi
    - |
      set -eu
      case "$$BUILDKITE_PARALLEL_JOB" in
        0)
          pytest -v -s tests/kernels/attention/test_attention_selector.py
          pytest -v -s tests/kernels/attention/test_cache.py -k nvfp4
          pytest -v -s tests/kernels/attention/test_flashinfer_trtllm_attention.py
          pytest -v -s tests/kernels/attention/test_flashinfer_mla_decode.py
          pytest -v -s tests/kernels/test_top_k_per_row.py
          pytest -v -s tests/kernels/quantization/test_silu_mul_nvfp4_quant.py
          pytest -v -s tests/kernels/quantization/test_flashinfer_scaled_mm.py
          pytest -v -s tests/kernels/moe/test_nvfp4_moe.py
          pytest -v -s tests/kernels/moe/test_ocp_mx_moe.py
          pytest -v -s tests/kernels/test_bf16x3_router_gemm_cutedsl.py
          pytest -v -s tests/kernels/test_ll_bf16_gemm.py
          ;;
        1)
          python3 examples/basic/offline_inference/chat.py
          pytest -v -s tests/kernels/attention/test_minimax_m3_msa_cutlass_sparse_decode.py
          pytest -v -s tests/kernels/test_fused_minimax_m3_qknorm_rope_kv_insert.py
          pytest -v -s tests/kernels/quantization/test_flashinfer_nvfp4_scaled_mm.py
          pytest -v -s tests/kernels/quantization/test_nvfp4_qutlass.py
          pytest -v -s tests/kernels/moe/test_mxfp4_moe.py
          pytest -v -s tests/kernels/moe/test_flashinfer_moe.py
          pytest -v -s tests/kernels/moe/test_cutedsl_moe.py
          pytest -v -s tests/models/quantization/test_nvfp4.py
          ;;
        2)
          pytest -v -s tests/kernels/attention/test_flashinfer.py -k 'not num_heads2'
          pytest -v -s tests/kernels/attention/test_cutlass_mla_decode.py
          pytest -v -s tests/kernels/quantization/test_cutlass_scaled_mm.py -k 'fp8'
          pytest -v -s tests/kernels/quantization/test_block_fp8.py::test_w8a8_block_fp8_cutlass_matmul
          pytest -v -s tests/kernels/quantization/test_nvfp4_quant.py
          pytest -v -s tests/kernels/quantization/test_nvfp4_scaled_mm.py
          pytest -v -s tests/kernels/quantization/test_mxfp4_qutlass.py
          pytest -v -s tests/kernels/moe/test_flashinfer.py
          pytest -v -s tests/kernels/moe/test_trtllm_nvfp4_moe.py
          pytest -v -s tests/kernels/mamba/test_gdn_prefill_cutedsl.py
          ;;
        *)
          echo "unexpected BUILDKITE_PARALLEL_JOB=$$BUILDKITE_PARALLEL_JOB" >&2
          exit 1
          ;;
      esac
  mirror:
    amd:
      label: ":amd: (MI355) NVFP4 Dense Emulation Shard %N"
      dind: false
      device: mi355_1
      num_devices: 1
      timeout_in_minutes: 10
      working_dir: "/vllm-workspace/tests"
      depends_on:
      - image-build-amd
      source_file_dependencies:
      - tests/kernels/quantization/test_nvfp4_emulation.py
      - tests/kernels/quantization/nvfp4_utils.py
      - tests/conftest.py
      - vllm/model_executor/kernels/linear/
      - vllm/model_executor/layers/quantization/utils/nvfp4_emulation_utils.py
      - vllm/config/kernel.py
      - vllm/platforms/
      - vllm/envs.py
      - vllm/utils/torch_utils.py
      - requirements/rocm-test.txt
      # W4A4 format numerics and replay via emulation. TODO: native packed
      # activation/KV-cache layouts and CUDA-only kernels need AMD backends.
      commands:
      - amd-smi
      - pytest -v -s kernels/quantization/test_nvfp4_emulation.py -k test_nvfp4_dense_emulation --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT

- label: ":nvidia: (DGX) Spark b12x Linear Kernels Nightly"
  key: b12x-linear-kernels-dgx-spark-nightly
  timeout_in_minutes: 60
  device: dgx-spark
  optional: true
  num_devices: 1
  depends_on:
  - arm64-image-build
  source_file_dependencies:
  - setup.py
  - vllm/config/kernel.py
  - vllm/envs.py
  - vllm/model_executor/kernels/linear/
  - vllm/model_executor/layers/fused_moe/b12x.py
  - vllm/model_executor/layers/fused_moe/config.py
  - vllm/model_executor/layers/fused_moe/oracle/mxfp4.py
  - vllm/model_executor/layers/fused_moe/oracle/nvfp4.py
  - vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/
  - vllm/model_executor/layers/quantization/inc/schemes/inc_mxfp4_moe.py
  - vllm/model_executor/layers/quantization/modelopt.py
  - vllm/model_executor/layers/quantization/mxfp4.py
  - vllm/model_executor/layers/quantization/utils/b12x_moe.py
  - vllm/model_executor/warmup/b12x_warmup.py
  - vllm/utils/b12x.py
  - tests/kernels/moe/test_b12x.py
  - tests/kernels/quantization/nvfp4_utils.py
  - tests/model_executor/kernels/test_b12x_linear.py
  - tests/model_executor/test_b12x_warmup.py
  - tests/kernels/quantization/test_block_fp8.py
  - tests/quantization/reference_mxfp4.py
  - tests/quantization/test_auto_round.py
  commands:
  - uv pip install --system b12x==1.3.0
  - pytest -v -s model_executor/kernels/test_b12x_linear.py
      model_executor/test_b12x_warmup.py
      kernels/quantization/test_block_fp8.py
      kernels/moe/test_b12x.py
      quantization/test_auto_round.py -k b12x

- label: ":nvidia: (H100) Helion Kernels Shard %N"
  key: kernels-helion-test
  timeout_in_minutes: 115
  device: h100
  source_file_dependencies:
  - vllm/utils/import_utils.py
  - tests/kernels/helion/
  commands:
    - pip install helion==1.4.0
    - pytest -v -s kernels/helion/ --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
  parallelism: 5
  mirror:
    amd:
      label: ":amd: (MI250) Helion Kernels Shard %N"
      dind: false
      device: mi250_1
      timeout_in_minutes: 45
      # Explicit so the base step's count does not multiply the mirror's replicas.
      parallelism: 2
      depends_on:
      - image-build-amd
      source_file_dependencies:
      - vllm/platforms/rocm.py
      - tools/benchmark_helion_kernels.py

 
- label: ":nvidia: (H100) FP8 MoE Kernels"
  key: kernels-fp8-moe-test-1xh100
  timeout_in_minutes: 40
  device: h100
  num_devices: 1
  optional: true
  commands:
    - pytest -v -s kernels/moe/test_cutlass_moe.py
    - pytest -v -s kernels/moe/test_flashinfer.py
    - pytest -v -s kernels/moe/test_gpt_oss_triton_kernels.py
    - pytest -v -s kernels/moe/test_modular_oai_triton_moe.py
    - pytest -v -s kernels/moe/test_moe.py
    # - pytest -v -s kernels/moe/test_block_fp8.py - failing on main
    - pytest -v -s kernels/moe/test_block_int8.py
    - pytest -v -s kernels/moe/test_triton_moe_no_act_mul.py
    - pytest -v -s kernels/moe/test_triton_moe_ptpc_fp8.py

- label: ":nvidia: (H100) DeepEP FP8 MoE Kernels"
  key: kernels-fp8-moe-test-2xh100
  timeout_in_minutes: 45
  device: h100
  num_devices: 2
  optional: true
  commands:
    - pytest -v -s kernels/moe/test_deepep_deepgemm_moe.py
    - pytest -v -s kernels/moe/test_deepep_moe.py
  mirror:
    amd:
      label: ":amd: (MI355) DeepEP FP8 MoE Kernels"
      dind: false
      device: mi355_2
      timeout_in_minutes: 55
      working_dir: "/vllm-workspace/tests"
      depends_on:
      - image-build-amd
      source_file_dependencies:
      - csrc/moe/
      - csrc/quantization/w8a8/cutlass/moe/
      - vllm/model_executor/layers/fused_moe/
      - tests/kernels/moe/test_deepep_moe.py
      - vllm/_aiter_ops.py
      - vllm/platforms/rocm.py
      - vllm/envs.py
      commands:
      - pytest -v -s kernels/moe/test_deepep_moe.py

- label: ":nvidia: (B200) FP4 MoE Kernels"
  key: kernels-fp4-moe-test-b200
  timeout_in_minutes: 25
  device: b200-k8s
  num_devices: 1
  optional: true
  commands:
    - pytest -v -s kernels/moe/test_cutedsl_moe.py
    - pytest -v -s kernels/moe/test_flashinfer_moe.py
    - pytest -v -s kernels/moe/test_nvfp4_moe.py
    - pytest -v -s kernels/moe/test_ocp_mx_moe.py
  mirror:
    amd:
      label: ":amd: (MI355 DPX) FP4 and INT4 MoE Kernels"
      dind: false
      device: mi355_dpx
      num_devices: 1
      timeout_in_minutes: 40
      optional: false
      working_dir: /vllm-workspace/tests
      depends_on:
      - image-build-amd
      source_file_dependencies:
      - csrc/moe/
      - csrc/rocm/
      - vllm/model_executor/layers/fused_moe/
      - vllm/model_executor/layers/quantization/
      - vllm/_aiter_ops.py
      - vllm/platforms/rocm.py
      - vllm/envs.py
      - vllm/config/
      - tests/kernels/moe/test_rocm_aiter_moe.py
      - tests/kernels/moe/test_ocp_mx_moe.py
      - tests/kernels/moe/test_flydsl_moe.py
      - tests/kernels/moe/utils.py
      - tests/kernels/utils.py
      commands:
      - pytest -v -s kernels/moe/test_rocm_aiter_moe.py -k 'mi350_mxfp4 or quant_scheme_support_matches_gfx950'
      - pytest -v -s kernels/moe/test_ocp_mx_moe.py::test_rocm_mxfp4_moe_oracle -k AITER_MXFP4
      - pytest -v -s kernels/moe/test_flydsl_moe.py


- label: ":nvidia: (H100) FusedMoE Layer Kernels"
  key: kernels-fusedmoe-layer-test-2-h100s
  timeout_in_minutes: 30
  device: h100
  num_devices: 2
  source_file_dependencies:
  - csrc/quantization/w8a8/cutlass/moe/
  - csrc/moe/
  - tests/kernels/moe
  - vllm/model_executor/layers/fused_moe/
  - vllm/model_executor/layers/quantization/
  - vllm/distributed/device_communicators/
  - vllm/config
  commands:
    - pytest -v -s kernels/moe/test_moe_layer.py
  mirror:
    amd:
      label: ":amd: (MI355) FusedMoE Layer Kernels"
      dind: false
      device: mi355_2
      timeout_in_minutes: 35
      working_dir: "/vllm-workspace/tests"
      depends_on:
      - image-build-amd
      source_file_dependencies:
      - csrc/moe/
      - csrc/rocm/
      - tests/kernels/moe
      - vllm/model_executor/layers/fused_moe/
      - vllm/model_executor/layers/quantization/
      - vllm/distributed/
      - vllm/config/
      - vllm/forward_context.py
      - vllm/v1/worker/workspace.py
      - vllm/utils/import_utils.py
      - vllm/utils/math_utils.py
      - vllm/utils/torch_utils.py
      - vllm/platforms/
      - vllm/_aiter_ops.py
      commands:
      # MoRI graph cases run in the Fault Tolerance group's healthy EP baseline.
      - pytest -v -s kernels/moe/test_moe_layer.py -k "not test_moe_layer_mori_graph"

- label: ":nvidia: (B200) FusedMoE Layer Kernels"
  key: kernels-fusedmoe-layer-test-2-b200s
  timeout_in_minutes: 90
  device: b200-k8s
  num_devices: 2
  source_file_dependencies:
  - csrc/quantization/w8a8/cutlass/moe/
  - csrc/moe/
  - tests/kernels/moe
  - vllm/model_executor/layers/fused_moe/
  - vllm/model_executor/layers/quantization/
  - vllm/distributed/device_communicators/
  - vllm/config
  commands:
    - pytest -v -s kernels/moe/test_moe_layer.py
    - pytest -v -s kernels/moe/test_deepep_v2_moe.py
  mirror:
    amd:
      label: ":amd: (MI355) MoRI EP Numerics"
      dind: false
      device: mi355_2
      num_devices: 2
      timeout_in_minutes: 20
      working_dir: "/vllm-workspace/tests"
      depends_on:
      - image-build-amd
      source_file_dependencies:
      - tests/kernels/moe/test_mori_moe.py
      - tests/kernels/moe/utils.py
      - tests/kernels/moe/modular_kernel_tools/parallel_utils.py
      - tests/kernels/utils.py
      - tests/utils.py
      - tests/conftest.py
      - vllm/utils/torch_utils.py
      - vllm/model_executor/layers/fused_moe/
      - vllm/model_executor/layers/quantization/
      - vllm/distributed/
      - vllm/config/
      - vllm/forward_context.py
      - vllm/v1/worker/workspace.py
      - vllm/platforms/
      - vllm/_aiter_ops.py
      - requirements/rocm-test.txt
      # Same-node MoRI/AITER BF16 and per-token/channel FP8. TODO: the
      # DeepEP v2 FlashInfer backend matrix has no direct AMD implementation.
      commands:
      - amd-smi
      - VLLM_ROCM_USE_AITER=1 VLLM_ROCM_USE_AITER_MOE=1 VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=0 pytest -v -s kernels/moe/test_mori_moe.py

- label: ":amd: (MI355 DPX) Native Quantization Kernels"
  key: amd-native-quantization-kernels-mi355
  dind: false
  device: mi355_dpx
  num_devices: 1
  timeout_in_minutes: 40
  working_dir: /vllm-workspace/tests
  depends_on:
  - image-build-amd
  source_file_dependencies:
  - csrc/rocm/
  - vllm/model_executor/layers/quantization/
  - vllm/model_executor/kernels/linear/
  - vllm/_aiter_ops.py
  - vllm/_custom_ops.py
  - vllm/platforms/rocm.py
  - vllm/envs.py
  - tests/kernels/quantization/test_rocm_mxfp4.py
  - tests/kernels/quantization/test_rocm_mxfp8_linear.py
  - tests/kernels/quantization/test_rocm_skinny_gemms.py
  - tests/quantization/reference_mxfp4.py
  - tests/kernels/quant_utils.py
  - tests/kernels/utils.py
  commands:
  - pytest -v -s kernels/quantization/test_rocm_mxfp4.py
  - pytest -v -s kernels/quantization/test_rocm_mxfp8_linear.py
  - pytest -v -s kernels/quantization/test_rocm_skinny_gemms.py -k wvsplitkrc

- label: ":amd: (MI355 DPX) FP8 MoE Kernels"
  key: amd-fp8-moe-kernels-mi355
  dind: false
  device: mi355_dpx
  num_devices: 1
  timeout_in_minutes: 40
  working_dir: /vllm-workspace/tests
  depends_on:
  - image-build-amd
  source_file_dependencies:
  - csrc/moe/
  - vllm/model_executor/layers/fused_moe/
  - tests/kernels/moe/test_deepep_moe.py
  - vllm/_aiter_ops.py
  - vllm/platforms/rocm.py
  - vllm/envs.py
  - tests/kernels/moe/test_gpt_oss_triton_kernels.py
  - tests/kernels/moe/test_modular_oai_triton_moe.py
  - tests/kernels/moe/test_moe.py
  - tests/kernels/moe/test_block_int8.py
  - tests/kernels/moe/test_triton_moe_no_act_mul.py
  - tests/kernels/moe/test_triton_moe_ptpc_fp8.py
  commands:
  - pytest -v -s kernels/moe/test_gpt_oss_triton_kernels.py
  - pytest -v -s kernels/moe/test_modular_oai_triton_moe.py
  - pytest -v -s kernels/moe/test_moe.py
  - pytest -v -s kernels/moe/test_block_int8.py
  - pytest -v -s kernels/moe/test_triton_moe_no_act_mul.py
  - pytest -v -s kernels/moe/test_triton_moe_ptpc_fp8.py

- label: ":amd: (MI355 DPX) Kernels"
  key: amd-kernels-mi355
  dind: false
  device: mi355_dpx
  num_devices: 1
  timeout_in_minutes: 25
  working_dir: /vllm-workspace/
  depends_on:
  - image-build-amd
  source_file_dependencies:
  - csrc/quantization/fp4/
  - csrc/attention/mla/
  - csrc/quantization/w8a8/cutlass/moe/
  - vllm/model_executor/layers/fused_moe/cutlass_moe.py
  - vllm/v1/attention/backends/triton_attn.py
  - vllm/v1/attention/backends/rocm_attn.py
  - vllm/v1/attention/backends/rocm_aiter_fa.py
  - vllm/v1/attention/backends/rocm_aiter_unified_attn.py
  - vllm/v1/attention/backends/mla/aiter_triton_mla.py
  - vllm/v1/attention/backends/mla/rocm_aiter_mla.py
  - vllm/v1/attention/selector.py
  - vllm/platforms/rocm.py
  - vllm/_aiter_ops.py
  - examples/basic/offline_inference/chat.py
  - tests/kernels/attention/test_attention_selector.py
  commands:
  - amd-smi
  - python3 examples/basic/offline_inference/chat.py --attention-backend TRITON_ATTN
  - pytest -v -s tests/kernels/attention/test_attention_selector.py