group: Models - Language
depends_on:
  - image-build
steps:
- label: ":nvidia: (H200 MIG 18GB) Language Models (Standard)"
  key: language-models-tests-standard
  timeout_in_minutes: 30
  device: h200_18gb
  source_file_dependencies:
  - vllm/
  - "!vllm/distributed/kv_transfer/"
  - tests/models/language
  commands:
    # Test standard language models, excluding a subset of slow tests
    - pip freeze | grep -E 'torch'
    - pytest -v -s models/language -m 'core_model and (not slow_test)'
  mirror:
    amd:
      label: ":amd: (MI355 DPX) Language Models (Standard)"
      dind: false
      device: mi355_dpx
      timeout_in_minutes: 60
      depends_on:
      - image-build-amd

- label: ":nvidia: (H200 MIG 35GB) Language Models (Extra Standard) Shard %N"
  device: h200_35gb
  key: language-models-tests-extra-standard
  timeout_in_minutes: 40
  source_file_dependencies:
  - vllm/model_executor/models/
  - tests/models/language/pooling/test_embedding.py
  - tests/models/language/generation/test_common.py
  - tests/models/language/pooling/test_classification.py
  commands:
    # Shard slow subset of standard language models tests. Only run when model
    # source is modified, or when specified test files are modified
    - pip freeze | grep -E 'torch'
    - pytest -v -s models/language -m 'core_model and slow_test' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
  parallelism: 2
  mirror:
    amd:
      label: ":amd: (MI355 DPX) Language Models (Extra Standard) Shard %N"
      dind: false
      device: mi355_dpx
      timeout_in_minutes: 40
      depends_on:
      - image-build-amd
      source_file_dependencies:
      - vllm/model_executor/models/
      - vllm/model_executor/model_loader/
      - vllm/model_executor/layers/
      - vllm/v1/attention/backends/
      - vllm/v1/attention/selector.py
      - tests/models/language/pooling/test_embedding.py
      - tests/models/language/generation/test_common.py
      - tests/models/language/pooling/test_classification.py
      - vllm/_aiter_ops.py
      - vllm/platforms/rocm.py
- label: ":nvidia: (H200 MIG 35GB) Language Models (Hybrid) Shard %N"
  device: h200_35gb
  key: language-models-tests-hybrid
  timeout_in_minutes: 65
  env:
    # GitHub intermittently rejects HTTP/2 upload-pack requests from the H200
    # CI image. Force Git's smart protocol to use HTTP/1.1 for source installs.
    GIT_CONFIG_COUNT: "1"
    GIT_CONFIG_KEY_0: http.version
    GIT_CONFIG_VALUE_0: HTTP/1.1
  source_file_dependencies:
  - vllm/
  - "!vllm/distributed/kv_transfer/"
  - tests/models/language/generation
  commands:
    # Install fast path packages for testing against transformers
    # torch>=2.14 nightly requires C++20 for ATen headers (pytorch/pytorch#178150);
    # mamba@v2.3.0 pins -std=c++17, so build from a patched checkout instead of the git URL.
    - rm -rf /tmp/mamba-src && git clone --depth 1 --branch v2.3.0 https://github.com/state-spaces/mamba /tmp/mamba-src && sed -i 's/-std=c++17/-std=c++20/g' /tmp/mamba-src/setup.py && MAMBA_FORCE_BUILD=TRUE uv pip install --system --no-build-isolation /tmp/mamba-src
    - CAUSAL_CONV1D_FORCE_BUILD=TRUE uv pip install --system --no-build-isolation 'git+https://github.com/Dao-AILab/causal-conv1d@v1.6.0'
    # Shard the hybrid language model tests that are numerically stable on Hopper.
    - pytest -v -s models/language/generation -m hybrid_model -k 'not granite-4.0-tiny-preview' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
  parallelism: 2
  mirror:
    amd:
      label: ":amd: (MI355 DPX) Language Models (Hybrid) Shard %N"
      dind: false
      device: mi355_dpx
      timeout_in_minutes: 65
      depends_on:
      - image-build-amd
      commands:
      - MAMBA_FORCE_BUILD=TRUE uv pip install --system --no-build-isolation 'git+https://github.com/AndreasKaratzas/mamba@fix-rocm-7.0-warp-size-constexpr'
      - CAUSAL_CONV1D_FORCE_BUILD=TRUE uv pip install --system --no-build-isolation 'git+https://github.com/Dao-AILab/causal-conv1d@v1.6.0'
      - pytest -v -s models/language/generation -m hybrid_model -k 'not granite-4.0-tiny-preview' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB

# Granite 4 hybrid generation is sensitive to hardware-specific Triton SSD
# autotuning (https://github.com/vllm-project/vllm/issues/25194). Keep this one
# correctness test on L4 until its H200 output matches the Transformers reference.
- label: ":nvidia: (L4) Granite Language Model Compatibility"
  key: language-models-tests-granite-l4-compatibility
  device: l4
  num_devices: 1
  timeout_in_minutes: 65
  source_file_dependencies:
  - vllm/
  - "!vllm/distributed/kv_transfer/"
  - tests/models/language/generation
  commands:
    # torch>=2.14 nightly requires C++20 for ATen headers (pytorch/pytorch#178150);
    # mamba@v2.3.0 pins -std=c++17, so build from a patched checkout instead of the git URL.
    - rm -rf /tmp/mamba-src && git clone --depth 1 --branch v2.3.0 https://github.com/state-spaces/mamba /tmp/mamba-src && sed -i 's/-std=c++17/-std=c++20/g' /tmp/mamba-src/setup.py && MAMBA_FORCE_BUILD=TRUE uv pip install --system --no-build-isolation /tmp/mamba-src
    - CAUSAL_CONV1D_FORCE_BUILD=TRUE uv pip install --system --no-build-isolation 'git+https://github.com/Dao-AILab/causal-conv1d@v1.6.0'
    - pytest -v -s models/language/generation -m hybrid_model -k 'granite-4.0-tiny-preview'
  mirror:
    amd:
      label: ":amd: (MI355 DPX) Granite Language Model Compatibility"
      dind: false
      device: mi355_dpx
      num_devices: 1
      timeout_in_minutes: 80
      working_dir: /vllm-workspace/tests
      depends_on:
      - image-build-amd
      source_file_dependencies:
      - vllm/
      - "!vllm/distributed/kv_transfer/"
      - tests/models/language/generation
      commands:
      - export GIT_CONFIG_COUNT=1
      - export GIT_CONFIG_KEY_0=http.version
      - export GIT_CONFIG_VALUE_0=HTTP/1.1
      - MAMBA_FORCE_BUILD=TRUE uv pip install --system --no-build-isolation 'git+https://github.com/AndreasKaratzas/mamba@fix-rocm-7.0-warp-size-constexpr'
      - CAUSAL_CONV1D_FORCE_BUILD=TRUE uv pip install --system --no-build-isolation 'git+https://github.com/Dao-AILab/causal-conv1d@v1.6.0'
      - pytest -v -s models/language/generation -m hybrid_model -k 'granite-4.0-tiny-preview'

- label: ":nvidia: (H200 MIG 35GB) Language Models (Extended Generation)"
  device: h200_35gb
  key: language-models-test-extended-generation
  timeout_in_minutes: 80
  optional: true
  source_file_dependencies:
  - vllm/
  - "!vllm/distributed/kv_transfer/"
  - tests/models/language/generation
  commands:
    # Install fast path packages for testing against transformers
    # torch>=2.14 nightly requires C++20 for ATen headers (pytorch/pytorch#178150);
    # mamba@v2.3.0 pins -std=c++17, so build from a patched checkout instead of the git URL.
    - rm -rf /tmp/mamba-src && git clone --depth 1 --branch v2.3.0 https://github.com/state-spaces/mamba /tmp/mamba-src && sed -i 's/-std=c++17/-std=c++20/g' /tmp/mamba-src/setup.py && MAMBA_FORCE_BUILD=TRUE uv pip install --system --no-build-isolation /tmp/mamba-src
    - CAUSAL_CONV1D_FORCE_BUILD=TRUE uv pip install --system --no-build-isolation 'git+https://github.com/Dao-AILab/causal-conv1d@v1.6.0'
    - pytest -v -s models/language/generation -m '(not core_model) and (not hybrid_model)'
  mirror:
    amd:
      label: ":amd: (MI355 DPX) Language Models (Extended Generation)"
      dind: false
      device: mi355_dpx
      timeout_in_minutes: 95
      depends_on:
      - image-build-amd
      commands:
      - MAMBA_FORCE_BUILD=TRUE uv pip install --system --no-build-isolation 'git+https://github.com/AndreasKaratzas/mamba@fix-rocm-7.0-warp-size-constexpr'
      - CAUSAL_CONV1D_FORCE_BUILD=TRUE uv pip install --system --no-build-isolation 'git+https://github.com/Dao-AILab/causal-conv1d@v1.6.0'
      - pytest -v -s models/language/generation -m '(not core_model) and (not hybrid_model)'

- label: ":nvidia: (H200 MIG 18GB) Language Models (PPL)"
  key: language-models-test-ppl
  timeout_in_minutes: 30
  device: h200_18gb
  optional: true
  source_file_dependencies:
  - vllm/
  - "!vllm/distributed/kv_transfer/"
  - tests/models/language/generation_ppl_test
  commands:
    - pytest -v -s models/language/generation_ppl_test
  mirror:
    amd:
      label: ":amd: (MI250) Language Models (PPL)"
      dind: false
      device: mi250_1
      timeout_in_minutes: 65
      depends_on:
      - image-build-amd
      source_file_dependencies:
      - vllm/model_executor/models/qwen3_5.py
      - vllm/model_executor/models/qwen3_5_mtp.py
      - vllm/model_executor/models/qwen2.py
      - vllm/model_executor/models/qwen3.py
      - vllm/model_executor/models/qwen3_next.py
      - vllm/model_executor/models/qwen3_next_mtp.py
      - vllm/model_executor/layers/fla/ops/
      - vllm/_aiter_ops.py
      - vllm/v1/attention/backends/triton_attn.py
      - vllm/v1/attention/backends/rocm_attn.py
      - vllm/v1/attention/backends/rocm_aiter_unified_attn.py
      - vllm/v1/attention/backends/rocm_aiter_fa.py
      - vllm/v1/attention/backends/flex_attention.py
      - vllm/v1/attention/ops/
      - vllm/platforms/rocm.py
      - tests/models/language/generation_ppl_test

- label: ":nvidia: (H200 MIG 35GB) Language Models (Extended Pooling)"
  device: h200_35gb
  key: language-models-test-extended-pooling
  automatic_shard: true
  timeout_in_minutes: 120
  optional: true
  source_file_dependencies:
  - vllm/
  - "!vllm/distributed/kv_transfer/"
  - tests/models/language/pooling
  commands:
    # Per-test timeout: a hung test fails with a stack dump instead of holding
    # the job until Buildkite cancels it, which also blocks the full run's
    # coverage collection. A whole shard takes 12-21 minutes.
    - pytest -v -s models/language/pooling -m 'not core_model' --timeout=900 --timeout-method=thread
  mirror:
    amd:
      parallelism: 4
      commands:
        - "pytest -v -s models/language/pooling -m 'not core_model' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB"
      label: ":amd: (MI355 DPX) Language Models (Extended Pooling) Shard %N"
      dind: false
      device: mi355_dpx
      timeout_in_minutes: 95
      depends_on:
      - image-build-amd

- label: ":nvidia: (H200 MIG 35GB) Language Models (MTEB)"
  key: language-models-test-mteb
  timeout_in_minutes: 68
  device: h200_35gb
  optional: true
  source_file_dependencies:
  - vllm/
  - "!vllm/distributed/kv_transfer/"
  - tests/models/language/pooling_mteb_test
  commands:
    - pytest -v -s models/language/pooling_mteb_test
  mirror:
    amd:
      label: ":amd: (MI250) Language Models (MTEB)"
      dind: false
      device: mi250_1
      timeout_in_minutes: 60
      depends_on:
      - image-build-amd