iHore/vLLM
8
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity
Files
b515648641227afe0d0fe92be7efac80471b295e
vLLM/recipes/models/Qwen/Qwen3.6-35B-A3B.yaml
T
Roy WangandClaude Opus 5.5 b515648641 [Doc] Host model recipes in recipes/ (#61302) 
Signed-off-by: esmeetu <jasonailu87@gmail.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-17 09:14:36 +00:00

382 lines
13 KiB
YAML
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
meta:
title: "Qwen3.6-35B-A3B"
slug: "qwen3.6-35b-a3b"
provider: "Qwen"
description: "Smaller Qwen3.6 multimodal MoE model (35B total / 3B active) with BF16, FP8, and NVIDIA NVFP4 variants"
date_added: 2026-04-16
date_updated: 2026-09-14
tasks:
- multimodal
- text
performance_headline: "Compact Qwen3.6 MoE with 3B active parameters — single-GPU FP8 or 2-4 GPU or 4x Intel Arc Pro B60/B70 BF16 serving"
related_recipes:
- "Qwen/Qwen3.5-397B-A17B"
hardware:
dgx_spark_gb10: verified
h100: verified
h200: verified
mi300x: verified
mi325x: verified
mi355x: verified
rtx_pro_6000: verified
dgx_station_gb300: verified
rtx_5090: verified
arc_pro_b60: verified
arc_pro_b70: verified
model:
model_id: "Qwen/Qwen3.6-35B-A3B"
min_vllm_version: "0.17.0"
default_frontend: rust
architecture: moe
parameter_count: "35B"
active_parameters: "3B"
context_length: 262144
base_args:
- "--trust-remote-code"
base_env: {}
features:
tool_calling:
description: "Enable automatic tool choice with Qwen3 Coder parser"
args:
- "--enable-auto-tool-choice"
- "--tool-call-parser"
- "qwen3_coder"
reasoning:
description: "Enable chain-of-thought reasoning with Qwen3 parser"
args:
- "--reasoning-parser"
- "qwen3"
spec_decoding:
description: "Multi-token prediction speculative decoding for lower latency"
default_mode: mtp
modes:
mtp:
label: "MTP"
description: "Built-in Multi-Token Prediction, 3 draft tokens."
args:
- "--speculative-config"
- '{"method":"mtp","num_speculative_tokens":3,"moe_backend":"triton"}'
hardware_overrides:
rtx_pro_6000:
args:
- "--speculative-config"
- '{"method":"mtp","num_speculative_tokens":4,"moe_backend":"flashinfer_cutlass"}'
rtx_5090:
args:
- "--speculative-config"
- '{"method":"mtp","num_speculative_tokens":4,"moe_backend":"flashinfer_cutlass"}'
dspark:
label: "DSpark"
description: "DSpark drafting with an externally trained draft model; the served checkpoint is unchanged."
variants:
- default
args:
- "--speculative-config"
- '{"model": "RedHatAI/Qwen3.6-35B-A3B-speculator.dspark", "num_speculative_tokens": 7, "method": "dspark"}'
text_only:
description: "Skip loading the vision encoder for text-only workloads — frees VRAM for KV cache. Mutually exclusive with encoder_parallel."
args:
- "--language-model-only"
encoder_parallel:
description: "Run the vision encoder in data-parallel mode — avoids TP comm overhead on the small encoder. Mutually exclusive with text_only."
args:
- "--mm-encoder-tp-mode"
- "data"
long_context:
description: "YaRN RoPE scaling past the native 262,144-token window to ~1,010,000 tokens. Static YaRN holds the scaling factor constant regardless of input length, so leave this off unless you actually serve long prompts; for ~524K, halve the factor to 2.0."
args:
- "--hf-overrides"
- '{"text_config": {"rope_parameters": {"mrope_interleaved": true, "mrope_section": [11, 11, 10], "rope_type": "yarn", "rope_theta": 10000000, "partial_rotary_factor": 0.25, "factor": 4.0, "original_max_position_embeddings": 262144}}}'
- "--max-model-len"
- "1010000"
env:
VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
opt_in_features:
- spec_decoding
- text_only
- long_context
variants:
default:
precision: bf16
vram_minimum_gb: 84
description: "Full precision BF16 — fits on 1x H200, 2x H100, 4x Intel Arc Pro B60 / B70"
hardware_overrides:
rtx_pro_6000:
extra_args:
- "--max-num-seqs"
- "8"
fp8:
model_id: "Qwen/Qwen3.6-35B-A3B-FP8"
precision: fp8
vram_minimum_gb: 42
description: "Qwen official FP8 checkpoint — single-GPU serving"
hardware_overrides:
rtx_pro_6000:
extra_env:
VLLM_USE_DEEP_GEMM: "0"
nvfp4:
model_id: "nvidia/Qwen3.6-35B-A3B-NVFP4"
precision: nvfp4
vram_minimum_gb: 21
tp: 1
min_vllm_version: "0.28.0"
docker_image: "vllm/vllm-openai:v0.28.0"
description: "NVIDIA ModelOpt NVFP4 checkpoint for Blackwell GPUs, including DGX Spark"
hardware_overrides:
dgx_spark_gb10:
extra_args:
- "--kv-cache-dtype"
- "fp8"
- "--attention-backend"
- "flashinfer"
- "--moe-backend"
- "marlin"
- "--gpu-memory-utilization"
- "0.5"
- "--max-model-len"
- "262144"
- "--max-num-seqs"
- "8"
- "--max-num-batched-tokens"
- "8192"
- "--enable-chunked-prefill"
- "--async-scheduling"
- "--enable-prefix-caching"
- "--load-format"
- "fastsafetensors"
dgx_station_gb300:
extra_args:
- "--kv-cache-dtype"
- "fp8"
- "--attention-backend"
- "flashinfer"
- "--moe-backend"
- "marlin"
- "--gpu-memory-utilization"
- "0.92"
- "--max-model-len"
- "262144"
- "--max-num-batched-tokens"
- "8192"
- "--enable-chunked-prefill"
- "--async-scheduling"
- "--enable-prefix-caching"
- "--load-format"
- "fastsafetensors"
rtx_pro_6000:
extra_args:
- "--quantization"
- "modelopt_fp4"
- "--kv-cache-dtype"
- "fp8"
- "--block-size"
- "128"
- "--moe-backend"
- "marlin"
- "--attention-backend"
- "flashinfer"
- "--attention-config"
- '{"use_trtllm_attention":true}'
- "--gpu-memory-utilization"
- "0.9"
- "--max-num-seqs"
- "8"
- "--max-num-batched-tokens"
- "8192"
- "--async-scheduling"
- "--enable-chunked-prefill"
- "--enable-prefix-caching"
- "--skip-mm-profiling"
- "--load-format"
- "fastsafetensors"
extra_env:
VLLM_HAS_FLASHINFER_CUBIN: "1"
rtx_5090:
extra_args:
- "--quantization"
- "modelopt_fp4"
- "--kv-cache-dtype"
- "fp8"
- "--block-size"
- "128"
- "--moe-backend"
- "marlin"
- "--attention-backend"
- "flashinfer"
- "--attention-config"
- '{"use_trtllm_attention":true}'
- "--gpu-memory-utilization"
- "0.85"
- "--max-model-len"
- "65536"
- "--max-num-seqs"
- "8"
- "--max-num-batched-tokens"
- "8192"
- "--enable-chunked-prefill"
- "--enable-prefix-caching"
- "--async-scheduling"
- "--skip-mm-profiling"
extra_env:
VLLM_HAS_FLASHINFER_CUBIN: "1"
redhat_nvfp4:
model_id: "RedHatAI/Qwen3.6-35B-A3B-NVFP4"
precision: nvfp4
label: "NVFP4 (RedHat)"
vram_minimum_gb: 21
description: "RedHatAI compressed-tensors NVFP4 quantization for Blackwell GPUs"
extra_args:
- "--kv-cache-dtype"
- "fp8"
redhat_fp8:
model_id: "RedHatAI/Qwen3.6-35B-A3B-FP8-dynamic"
precision: fp8
label: "FP8 Dynamic (RedHat)"
vram_minimum_gb: 42
description: "RedHatAI FP8 (E4M3) with dynamic per-token activation quantization"
extra_args:
- "--kv-cache-dtype"
- "fp8"
redhat_fp8_plain:
model_id: "RedHatAI/Qwen3.6-35B-A3B-FP8"
precision: fp8
label: "FP8 (RedHat)"
vram_minimum_gb: 42
description: "RedHatAI FP8 quantization"
extra_args:
- "--kv-cache-dtype"
- "fp8"
compatible_strategies:
- single_node_tp
- single_node_tep
- single_node_dep
- multi_node_tp
- multi_node_tp_pp
- multi_node_dep
- multi_node_tep
hardware_overrides:
amd:
extra_args: []
extra_env:
VLLM_ROCM_USE_AITER: "1"
xpu:
extra_args:
- "--enforce-eager"
strategy_overrides: {}
guide: |
## Overview
[Qwen3.6-35B-A3B](https://huggingface.co/Qwen/Qwen3.6-35B-A3B) is the smaller sibling
of Qwen3.5, sharing the same gated-delta-networks MoE architecture but with 35B total
parameters and 3B activated (256 experts, 8 routed + 1 shared). The recipe covers the
BF16 base model, Qwen's official FP8 checkpoint, and NVIDIA's ModelOpt NVFP4
checkpoint.
## Prerequisites
- **vLLM version:** >= 0.17.0
- **NVFP4 vLLM version:** >= 0.28.0. 0.24.0 loads the checkpoint, but
FlashInfer only selects its XQA decode kernel on SM120 GPUs from 0.28.0.
- **Hardware (BF16):** 1x H200, 2x H100, 4x Intel Arc Pro B60 / B70
- **Hardware (FP8):** single H100/H200 or 1x MI300X/MI325X/MI355X
- **Hardware (NVFP4):** NVIDIA Blackwell GPUs, including DGX Spark (GB10)
## NVFP4 on Blackwell
All NVFP4 tuning lives in `variants.nvfp4.hardware_overrides`, keyed by GPU
profile, so the exact flag set for a box is readable straight from the recipe
YAML (or from `by_hardware` in the JSON API) without going through the
command builder.
Four profiles are tuned and verified. DGX Spark (GB10) and DGX Station
(GB300) run the checkpoint on the default loader with FlashInfer attention,
`--moe-backend marlin` and a 3-token MTP draft on Triton; Spark shares
unified memory with the host, so `--gpu-memory-utilization` is capped at 0.5.
The SM120 parts (RTX PRO 6000, RTX 5090) load through `--quantization
modelopt_fp4` with FlashInfer's TRT-LLM attention kernels, which need
`--block-size 128` and `VLLM_HAS_FLASHINFER_CUBIN=1`, and sustain a deeper
4-token MTP draft on `flashinfer_cutlass`. The RTX 5090's 32 GB caps context
at 64K.
Datacenter Blackwell (B200/GB200/B300/GB300) intentionally carries no
overrides — it runs vLLM's own NVFP4 defaults, which select the native FP4
path rather than the Marlin W4A16 kernel the workstation parts need.
### Measured on DGX Spark (GB10)
The GB10 profile above was re-measured on 2026-08-31 with aiperf 0.12.0 on
vLLM 0.28.0 (Ubuntu 24.04.4, driver 580.159.03, CUDA 13.0), one GB10 at TP1,
serving from `vllm/vllm-openai:v0.28.0`. On SPEED-Bench at 8K input / 256
output tokens and concurrency 1, it sustained 6,265 prefill tok/s and 97.7
decode tok/s at a 3.95 s mean end-to-end latency. The flag set is unchanged —
the sweep confirms the tuning rather than revising it, so the command the
builder emits for GB10 is the one that produced these numbers.
### Docker (Intel XPU B60 / B70)
Validated on 4× Intel Arc Pro B60 / B70 (B60 24 GB, B70 32 GB per card) with the
official vLLM XPU image `vllm/vllm-openai-xpu:latest`.
```bash
docker run --device /dev/dri \
-v /dev/dri/by-path:/dev/dri/by-path --shm-size=16g \
--privileged --ipc=host -p 8000:8000 \
-v ~/.cache/huggingface:/root/.cache/huggingface \
vllm/vllm-openai-xpu:latest \
Qwen/Qwen3.6-35B-A3B \
--reasoning-parser qwen3 \
--tensor-parallel-size 4 \
--max-model-len 8192 \
--enforce-eager
```
## Processing Ultra-Long Texts
Qwen3.6-35B-A3B natively supports `262,144` tokens. To serve longer inputs, turn on the
**Long Context** feature above: it applies the model card's YaRN RoPE
configuration via `--hf-overrides` and raises `--max-model-len` to ~1,010,000 tokens.
Static YaRN holds the scaling factor constant regardless of input length, so
leave it off unless you actually serve long prompts; for a ~524K workload,
halve the card's `factor` to `2.0`.
See the [model card](https://huggingface.co/Qwen/Qwen3.6-35B-A3B#processing-ultra-long-texts) for the full parameter reference.
## Client Usage
```python
from openai import OpenAI
client = OpenAI(api_key="EMPTY", base_url="http://localhost:8000/v1")
resp = client.chat.completions.create(
model="Qwen/Qwen3.6-35B-A3B",
messages=[{"role": "user", "content": "Explain gated delta networks in one paragraph."}],
max_tokens=512,
)
print(resp.choices[0].message.content)
```
## Troubleshooting
- **CUDA graph / Mamba cache size error:** reduce `--max-cudagraph-capture-size`
(default 512). See [vLLM PR #34571](https://github.com/vllm-project/vllm/pull/34571).
- **Reasoning disable:** add `--default-chat-template-kwargs '{"enable_thinking": false}'`.
- **Prefix Caching (Mamba):** currently experimental in "align" mode.
## References
- [Qwen3.6-35B-A3B on Hugging Face](https://huggingface.co/Qwen/Qwen3.6-35B-A3B)
- [FP8 checkpoint](https://huggingface.co/Qwen/Qwen3.6-35B-A3B-FP8)
- [NVIDIA NVFP4 checkpoint](https://huggingface.co/nvidia/Qwen3.6-35B-A3B-NVFP4)
- [Qwen3.5 recipe (sibling 397B-A17B flagship)](../Qwen3.5-397B-A17B)
Reference in New Issue View Git Blame Copy Permalink