meta:
  title: "Qwen3.6-35B-A3B"
  slug: "qwen3.6-35b-a3b"
  provider: "Qwen"
  description: "Smaller Qwen3.6 multimodal MoE model (35B total / 3B active) with BF16, FP8, and NVIDIA NVFP4 variants"
  date_added: 2026-04-16
  date_updated: 2026-09-14
  tasks:
    - multimodal
    - text
  performance_headline: "Compact Qwen3.6 MoE with 3B active parameters — single-GPU FP8 or 2-4 GPU or 4x Intel Arc Pro B60/B70 BF16 serving"
  related_recipes:
    - "Qwen/Qwen3.5-397B-A17B"
  hardware:
    dgx_spark_gb10: verified
    h100: verified
    h200: verified
    mi300x: verified
    mi325x: verified
    mi355x: verified
    rtx_pro_6000: verified
    dgx_station_gb300: verified
    rtx_5090: verified
    arc_pro_b60: verified
    arc_pro_b70: verified

model:
  model_id: "Qwen/Qwen3.6-35B-A3B"
  min_vllm_version: "0.17.0"
  default_frontend: rust
  architecture: moe
  parameter_count: "35B"
  active_parameters: "3B"
  context_length: 262144
  base_args:
    - "--trust-remote-code"
  base_env: {}

features:
  tool_calling:
    description: "Enable automatic tool choice with Qwen3 Coder parser"
    args:
      - "--enable-auto-tool-choice"
      - "--tool-call-parser"
      - "qwen3_coder"
  reasoning:
    description: "Enable chain-of-thought reasoning with Qwen3 parser"
    args:
      - "--reasoning-parser"
      - "qwen3"
  spec_decoding:
    description: "Multi-token prediction speculative decoding for lower latency"
    default_mode: mtp
    modes:
      mtp:
        label: "MTP"
        description: "Built-in Multi-Token Prediction, 3 draft tokens."
        args:
          - "--speculative-config"
          - '{"method":"mtp","num_speculative_tokens":3,"moe_backend":"triton"}'
        hardware_overrides:
          rtx_pro_6000:
            args:
              - "--speculative-config"
              - '{"method":"mtp","num_speculative_tokens":4,"moe_backend":"flashinfer_cutlass"}'
          rtx_5090:
            args:
              - "--speculative-config"
              - '{"method":"mtp","num_speculative_tokens":4,"moe_backend":"flashinfer_cutlass"}'
      dspark:
        label: "DSpark"
        description: "DSpark drafting with an externally trained draft model; the served checkpoint is unchanged."
        variants:
          - default
        args:
          - "--speculative-config"
          - '{"model": "RedHatAI/Qwen3.6-35B-A3B-speculator.dspark", "num_speculative_tokens": 7, "method": "dspark"}'
  text_only:
    description: "Skip loading the vision encoder for text-only workloads — frees VRAM for KV cache. Mutually exclusive with encoder_parallel."
    args:
      - "--language-model-only"
  encoder_parallel:
    description: "Run the vision encoder in data-parallel mode — avoids TP comm overhead on the small encoder. Mutually exclusive with text_only."
    args:
      - "--mm-encoder-tp-mode"
      - "data"
  long_context:
    description: "YaRN RoPE scaling past the native 262,144-token window to ~1,010,000 tokens. Static YaRN holds the scaling factor constant regardless of input length, so leave this off unless you actually serve long prompts; for ~524K, halve the factor to 2.0."
    args:
      - "--hf-overrides"
      - '{"text_config": {"rope_parameters": {"mrope_interleaved": true, "mrope_section": [11, 11, 10], "rope_type": "yarn", "rope_theta": 10000000, "partial_rotary_factor": 0.25, "factor": 4.0, "original_max_position_embeddings": 262144}}}'
      - "--max-model-len"
      - "1010000"
    env:
      VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"

opt_in_features:
  - spec_decoding
  - text_only
  - long_context

variants:
  default:
    precision: bf16
    vram_minimum_gb: 84
    description: "Full precision BF16 — fits on 1x H200, 2x H100, 4x Intel Arc Pro B60 / B70"
    hardware_overrides:
      rtx_pro_6000:
        extra_args:
          - "--max-num-seqs"
          - "8"
  fp8:
    model_id: "Qwen/Qwen3.6-35B-A3B-FP8"
    precision: fp8
    vram_minimum_gb: 42
    description: "Qwen official FP8 checkpoint — single-GPU serving"
    hardware_overrides:
      rtx_pro_6000:
        extra_env:
          VLLM_USE_DEEP_GEMM: "0"
  nvfp4:
    model_id: "nvidia/Qwen3.6-35B-A3B-NVFP4"
    precision: nvfp4
    vram_minimum_gb: 21
    tp: 1
    min_vllm_version: "0.28.0"
    docker_image: "vllm/vllm-openai:v0.28.0"
    description: "NVIDIA ModelOpt NVFP4 checkpoint for Blackwell GPUs, including DGX Spark"
    hardware_overrides:
      dgx_spark_gb10:
        extra_args:
          - "--kv-cache-dtype"
          - "fp8"
          - "--attention-backend"
          - "flashinfer"
          - "--moe-backend"
          - "marlin"
          - "--gpu-memory-utilization"
          - "0.5"
          - "--max-model-len"
          - "262144"
          - "--max-num-seqs"
          - "8"
          - "--max-num-batched-tokens"
          - "8192"
          - "--enable-chunked-prefill"
          - "--async-scheduling"
          - "--enable-prefix-caching"
          - "--load-format"
          - "fastsafetensors"
      dgx_station_gb300:
        extra_args:
          - "--kv-cache-dtype"
          - "fp8"
          - "--attention-backend"
          - "flashinfer"
          - "--moe-backend"
          - "marlin"
          - "--gpu-memory-utilization"
          - "0.92"
          - "--max-model-len"
          - "262144"
          - "--max-num-batched-tokens"
          - "8192"
          - "--enable-chunked-prefill"
          - "--async-scheduling"
          - "--enable-prefix-caching"
          - "--load-format"
          - "fastsafetensors"
      rtx_pro_6000:
        extra_args:
          - "--quantization"
          - "modelopt_fp4"
          - "--kv-cache-dtype"
          - "fp8"
          - "--block-size"
          - "128"
          - "--moe-backend"
          - "marlin"
          - "--attention-backend"
          - "flashinfer"
          - "--attention-config"
          - '{"use_trtllm_attention":true}'
          - "--gpu-memory-utilization"
          - "0.9"
          - "--max-num-seqs"
          - "8"
          - "--max-num-batched-tokens"
          - "8192"
          - "--async-scheduling"
          - "--enable-chunked-prefill"
          - "--enable-prefix-caching"
          - "--skip-mm-profiling"
          - "--load-format"
          - "fastsafetensors"
        extra_env:
          VLLM_HAS_FLASHINFER_CUBIN: "1"
      rtx_5090:
        extra_args:
          - "--quantization"
          - "modelopt_fp4"
          - "--kv-cache-dtype"
          - "fp8"
          - "--block-size"
          - "128"
          - "--moe-backend"
          - "marlin"
          - "--attention-backend"
          - "flashinfer"
          - "--attention-config"
          - '{"use_trtllm_attention":true}'
          - "--gpu-memory-utilization"
          - "0.85"
          - "--max-model-len"
          - "65536"
          - "--max-num-seqs"
          - "8"
          - "--max-num-batched-tokens"
          - "8192"
          - "--enable-chunked-prefill"
          - "--enable-prefix-caching"
          - "--async-scheduling"
          - "--skip-mm-profiling"
        extra_env:
          VLLM_HAS_FLASHINFER_CUBIN: "1"

  redhat_nvfp4:
    model_id: "RedHatAI/Qwen3.6-35B-A3B-NVFP4"
    precision: nvfp4
    label: "NVFP4 (RedHat)"
    vram_minimum_gb: 21
    description: "RedHatAI compressed-tensors NVFP4 quantization for Blackwell GPUs"
    extra_args:
      - "--kv-cache-dtype"
      - "fp8"

  redhat_fp8:
    model_id: "RedHatAI/Qwen3.6-35B-A3B-FP8-dynamic"
    precision: fp8
    label: "FP8 Dynamic (RedHat)"
    vram_minimum_gb: 42
    description: "RedHatAI FP8 (E4M3) with dynamic per-token activation quantization"
    extra_args:
      - "--kv-cache-dtype"
      - "fp8"

  redhat_fp8_plain:
    model_id: "RedHatAI/Qwen3.6-35B-A3B-FP8"
    precision: fp8
    label: "FP8 (RedHat)"
    vram_minimum_gb: 42
    description: "RedHatAI FP8 quantization"
    extra_args:
      - "--kv-cache-dtype"
      - "fp8"

compatible_strategies:
  - single_node_tp
  - single_node_tep
  - single_node_dep
  - multi_node_tp
  - multi_node_tp_pp
  - multi_node_dep
  - multi_node_tep

hardware_overrides:
  amd:
    extra_args: []
    extra_env:
      VLLM_ROCM_USE_AITER: "1"
  xpu:
    extra_args:
      - "--enforce-eager"

strategy_overrides: {}

guide: |
  ## Overview

  [Qwen3.6-35B-A3B](https://huggingface.co/Qwen/Qwen3.6-35B-A3B) is the smaller sibling
  of Qwen3.5, sharing the same gated-delta-networks MoE architecture but with 35B total
  parameters and 3B activated (256 experts, 8 routed + 1 shared). The recipe covers the
  BF16 base model, Qwen's official FP8 checkpoint, and NVIDIA's ModelOpt NVFP4
  checkpoint.

  ## Prerequisites

  - **vLLM version:** >= 0.17.0
  - **NVFP4 vLLM version:** >= 0.28.0. 0.24.0 loads the checkpoint, but
    FlashInfer only selects its XQA decode kernel on SM120 GPUs from 0.28.0.
  - **Hardware (BF16):** 1x H200, 2x H100, 4x Intel Arc Pro B60 / B70
  - **Hardware (FP8):** single H100/H200 or 1x MI300X/MI325X/MI355X
  - **Hardware (NVFP4):** NVIDIA Blackwell GPUs, including DGX Spark (GB10)

  ## NVFP4 on Blackwell

  All NVFP4 tuning lives in `variants.nvfp4.hardware_overrides`, keyed by GPU
  profile, so the exact flag set for a box is readable straight from the recipe
  YAML (or from `by_hardware` in the JSON API) without going through the
  command builder.

  Four profiles are tuned and verified. DGX Spark (GB10) and DGX Station
  (GB300) run the checkpoint on the default loader with FlashInfer attention,
  `--moe-backend marlin` and a 3-token MTP draft on Triton; Spark shares
  unified memory with the host, so `--gpu-memory-utilization` is capped at 0.5.
  The SM120 parts (RTX PRO 6000, RTX 5090) load through `--quantization
  modelopt_fp4` with FlashInfer's TRT-LLM attention kernels, which need
  `--block-size 128` and `VLLM_HAS_FLASHINFER_CUBIN=1`, and sustain a deeper
  4-token MTP draft on `flashinfer_cutlass`. The RTX 5090's 32 GB caps context
  at 64K.

  Datacenter Blackwell (B200/GB200/B300/GB300) intentionally carries no
  overrides — it runs vLLM's own NVFP4 defaults, which select the native FP4
  path rather than the Marlin W4A16 kernel the workstation parts need.

  ### Measured on DGX Spark (GB10)

  The GB10 profile above was re-measured on 2026-08-31 with aiperf 0.12.0 on
  vLLM 0.28.0 (Ubuntu 24.04.4, driver 580.159.03, CUDA 13.0), one GB10 at TP1,
  serving from `vllm/vllm-openai:v0.28.0`. On SPEED-Bench at 8K input / 256
  output tokens and concurrency 1, it sustained 6,265 prefill tok/s and 97.7
  decode tok/s at a 3.95 s mean end-to-end latency. The flag set is unchanged —
  the sweep confirms the tuning rather than revising it, so the command the
  builder emits for GB10 is the one that produced these numbers.

  ### Docker (Intel XPU B60 / B70)

  Validated on 4× Intel Arc Pro B60 / B70 (B60 24 GB, B70 32 GB per card) with the
  official vLLM XPU image `vllm/vllm-openai-xpu:latest`.

  ```bash
  docker run --device /dev/dri \
    -v /dev/dri/by-path:/dev/dri/by-path --shm-size=16g \
    --privileged --ipc=host -p 8000:8000 \
    -v ~/.cache/huggingface:/root/.cache/huggingface \
    vllm/vllm-openai-xpu:latest \
    Qwen/Qwen3.6-35B-A3B \
    --reasoning-parser qwen3 \
    --tensor-parallel-size 4 \
    --max-model-len 8192 \
    --enforce-eager
  ```

  ## Processing Ultra-Long Texts

  Qwen3.6-35B-A3B natively supports `262,144` tokens. To serve longer inputs, turn on the
  **Long Context** feature above: it applies the model card's YaRN RoPE
  configuration via `--hf-overrides` and raises `--max-model-len` to ~1,010,000 tokens.
  Static YaRN holds the scaling factor constant regardless of input length, so
  leave it off unless you actually serve long prompts; for a ~524K workload,
  halve the card's `factor` to `2.0`.

  See the [model card](https://huggingface.co/Qwen/Qwen3.6-35B-A3B#processing-ultra-long-texts) for the full parameter reference.

  ## Client Usage

  ```python
  from openai import OpenAI

  client = OpenAI(api_key="EMPTY", base_url="http://localhost:8000/v1")
  resp = client.chat.completions.create(
      model="Qwen/Qwen3.6-35B-A3B",
      messages=[{"role": "user", "content": "Explain gated delta networks in one paragraph."}],
      max_tokens=512,
  )
  print(resp.choices[0].message.content)
  ```

  ## Troubleshooting

  - **CUDA graph / Mamba cache size error:** reduce `--max-cudagraph-capture-size`
    (default 512). See [vLLM PR #34571](https://github.com/vllm-project/vllm/pull/34571).
  - **Reasoning disable:** add `--default-chat-template-kwargs '{"enable_thinking": false}'`.
  - **Prefix Caching (Mamba):** currently experimental in "align" mode.

  ## References

  - [Qwen3.6-35B-A3B on Hugging Face](https://huggingface.co/Qwen/Qwen3.6-35B-A3B)
  - [FP8 checkpoint](https://huggingface.co/Qwen/Qwen3.6-35B-A3B-FP8)
  - [NVIDIA NVFP4 checkpoint](https://huggingface.co/nvidia/Qwen3.6-35B-A3B-NVFP4)
  - [Qwen3.5 recipe (sibling 397B-A17B flagship)](../Qwen3.5-397B-A17B)