Roy WangandClaude Opus 5.5
b515648641
Signed-off-by: esmeetu <jasonailu87@gmail.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
431 lines
15 KiB
YAML
431 lines
15 KiB
YAML
meta:
|
||
title: "Qwen3.8-Flash-Next"
|
||
slug: "qwen3.8-flash-next"
|
||
provider: "Qwen"
|
||
description: "Qwen4 architecture preview with a 125B-parameter main model, supplemented by an additional 51B N-gram embeddings, with 6B parameters activated per token."
|
||
date_updated: 2026-10-13
|
||
tasks:
|
||
- multimodal
|
||
- text
|
||
performance_headline: "Qwen4 architecture preview with 6B active parameters and efficient 262K context"
|
||
related_recipes:
|
||
- "Qwen/Qwen3-Next-80B-A3B-Instruct"
|
||
hardware:
|
||
h200: verified
|
||
h100: verified
|
||
gb200: verified
|
||
gb300: verified
|
||
mi355x: verified
|
||
rtx_pro_6000_4x: verified
|
||
dgx_station_gb300: verified
|
||
|
||
model:
|
||
model_id: "Qwen/Qwen3.8-Flash-Next"
|
||
min_vllm_version: "0.30.0"
|
||
default_frontend: rust
|
||
docker_image:
|
||
nvidia: "vllm/vllm-openai:nightly"
|
||
amd: "vllm/vllm-openai-rocm:nightly"
|
||
nightly_required: true
|
||
install:
|
||
pip:
|
||
note: "Nightly wheels are recommended and verified by this recipe; stable vLLM ≥0.30.0 also runs the model."
|
||
docker:
|
||
note: "Use the vLLM nightly image."
|
||
architecture: moe
|
||
parameter_count: "176B"
|
||
active_parameters: "6B"
|
||
context_length: 262144
|
||
base_args:
|
||
- "--max-num-seqs"
|
||
- "256"
|
||
- "--gpu-memory-utilization"
|
||
- "0.90"
|
||
- "--enable-prefix-caching"
|
||
base_env: {}
|
||
|
||
dependencies: []
|
||
|
||
features:
|
||
tool_calling:
|
||
description: "Enable automatic tool choice with the Qwen3 Coder parser"
|
||
args:
|
||
- "--enable-auto-tool-choice"
|
||
- "--tool-call-parser"
|
||
- "qwen3_coder"
|
||
reasoning:
|
||
description: "Extract thinking output with the Qwen3 reasoning parser"
|
||
args:
|
||
- "--reasoning-parser"
|
||
- "qwen3"
|
||
spec_decoding:
|
||
description: "Use the checkpoint's Multi-Token Prediction module for faster decoding"
|
||
args:
|
||
- "--speculative-config"
|
||
- '{"method":"mtp","num_speculative_tokens":3}'
|
||
text_only:
|
||
description: "Skip the vision encoder for text-only workloads and leave more memory for KV cache"
|
||
args:
|
||
- "--language-model-only"
|
||
long_context:
|
||
description: "YaRN RoPE scaling past the native 262,144-token window to 1,000,000 tokens. Static YaRN holds the scaling factor constant regardless of input length, so leave this off unless you actually serve long prompts; for ~524K, halve the factor to 2.0."
|
||
args:
|
||
- "--hf-overrides"
|
||
- '{"text_config": {"rope_parameters": {"mrope_interleaved": true, "mrope_section": [11, 11, 10], "rope_type": "yarn", "rope_theta": 10000000, "partial_rotary_factor": 0.25, "factor": 4.0, "original_max_position_embeddings": 262144}}}'
|
||
- "--max-model-len"
|
||
- "1000000"
|
||
env:
|
||
VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
|
||
|
||
opt_in_features:
|
||
- text_only
|
||
- long_context
|
||
|
||
hardware_opt_in_features:
|
||
rtx_pro_6000_4x:
|
||
- spec_decoding
|
||
|
||
variants:
|
||
default:
|
||
model_id: "Inferact/Qwen3.8-Flash-Next-NVFP4"
|
||
precision: nvfp4
|
||
vram_minimum_gb: 130
|
||
description: "Inferact quantized NVFP4 weights with per-layer embeddings in nvfp4 too"
|
||
hardware_overrides:
|
||
dgx_station_gb300:
|
||
extra_args:
|
||
- "--max-num-seqs"
|
||
- "32"
|
||
- "--gpu-memory-utilization"
|
||
- "0.95"
|
||
- "--max-num-batched-tokens"
|
||
- "8192"
|
||
- "--enable-prefix-caching"
|
||
- "--distributed-executor-backend"
|
||
- "mp"
|
||
- "-cc.mode"
|
||
- "none"
|
||
- "-cc.cudagraph_mode"
|
||
- "full_decode_only"
|
||
rtx_pro_6000_4x:
|
||
extra_args:
|
||
- "--max-num-seqs"
|
||
- "16"
|
||
- "--gpu-memory-utilization"
|
||
- "0.95"
|
||
- "--max-num-batched-tokens"
|
||
- "8192"
|
||
- "--enable-prefix-caching"
|
||
- "--moe-backend"
|
||
- "marlin"
|
||
fp8:
|
||
model_id: "Qwen/Qwen3.8-Flash-Next-FP8"
|
||
precision: fp8
|
||
vram_minimum_gb: 250
|
||
description: "Official dynamic FP8 checkpoint with blockwise weight quantization"
|
||
hardware_overrides:
|
||
dgx_station_gb300:
|
||
extra_args:
|
||
- "--max-num-seqs"
|
||
- "32"
|
||
- "--gpu-memory-utilization"
|
||
- "0.95"
|
||
- "--max-num-batched-tokens"
|
||
- "8192"
|
||
- "--enable-prefix-caching"
|
||
- "--distributed-executor-backend"
|
||
- "mp"
|
||
- "-cc.mode"
|
||
- "none"
|
||
- "-cc.cudagraph_mode"
|
||
- "full_decode_only"
|
||
rtx_pro_6000_4x:
|
||
extra_args:
|
||
- "--max-num-seqs"
|
||
- "16"
|
||
- "--gpu-memory-utilization"
|
||
- "0.95"
|
||
- "--max-num-batched-tokens"
|
||
- "8192"
|
||
- "--enable-prefix-caching"
|
||
bf16:
|
||
model_id: "Qwen/Qwen3.8-Flash-Next"
|
||
precision: bf16
|
||
vram_minimum_gb: 423
|
||
description: "Official BF16 checkpoint"
|
||
|
||
# TP+EP is the default deployment; it benchmarks consistently faster than
|
||
# plain TP for this model, which stays offered as an alternative.
|
||
default_strategy: single_node_tep
|
||
|
||
compatible_strategies:
|
||
- single_node_tp
|
||
- single_node_tep
|
||
- single_node_dep
|
||
|
||
kv_offload_support:
|
||
offloading_cpu: verified
|
||
offloading_fs: verified
|
||
|
||
hardware_overrides:
|
||
hopper:
|
||
extra_args:
|
||
- "--moe-backend"
|
||
- "triton"
|
||
- "--gpu-memory-utilization"
|
||
- "0.85"
|
||
- "--kv-cache-dtype"
|
||
- "fp8"
|
||
- "--attention-config.indexer_kv_dtype"
|
||
- "fp8"
|
||
extra_env: {}
|
||
blackwell:
|
||
extra_args:
|
||
- "--kv-cache-dtype"
|
||
- "fp8"
|
||
- "--attention-config.indexer_kv_dtype"
|
||
- "fp8"
|
||
strategy_overrides:
|
||
single_node_tp:
|
||
tp: 4
|
||
|
||
guide: |
|
||
## Overview
|
||
|
||
[Qwen3.8-Flash-Next](https://huggingface.co/Qwen/Qwen3.8-Flash-Next) is a
|
||
multimodal, ultra-sparse Mixture-of-Experts model. It has 125B parameters,
|
||
including an additional 51B N-gram embedding table, while activating 6B
|
||
parameters per token.
|
||
|
||
The architecture combines four main ideas:
|
||
|
||
- **GDN + QSA:** three of every four layers use Gated DeltaNet to compress history;
|
||
the fourth uses Qwen Sparse Attention for precise long-range retrieval.
|
||
- **Gated Residual:** four residual branches dynamically control cross-layer reads
|
||
and writes.
|
||
- **N-gram Embedding:** a 51B lookup memory adds capacity with little per-token
|
||
compute and can be asynchronously offloaded to host memory. (note that offload
|
||
currently only runs on Nvidia devices)
|
||
- **MTP:** the built-in Multi-Token Prediction module supports speculative decoding.
|
||
|
||
The checkpoint natively supports 262,144 tokens. Qwen reports that QSA reaches up
|
||
to 10.2x prefill and 6.6x decode attention-kernel speedups at one million tokens.
|
||
On GB300, TEP4 is the recommended full-tray configuration. On an 8x H200 node,
|
||
use TEP8 for the official FP8 checkpoint; plain TP8 is incompatible with its
|
||
128-wide quantization blocks.
|
||
|
||
## Prerequisites
|
||
|
||
- **Runtime:** use the nightly image, `vllm/vllm-openai:nightly` on NVIDIA or
|
||
`vllm/vllm-openai-rocm:nightly` on AMD, or the nightly PyPI wheels. Stable
|
||
≥0.30.0 runs the model, but this recipe is tuned and verified on nightly.
|
||
- **Hardware:** NVIDIA CUDA and AMD ROCm instructions are provided below.
|
||
- **FP8 memory:** the checkpoint is 172.78 GiB; it fits on a single GB300 GPU
|
||
(288 GB HBM3e).
|
||
- **BF16 memory:** the checkpoint is 335.28 GiB. The validated TP2 configuration
|
||
used about 190 GiB per GB300 GPU.
|
||
- **Host memory for N-gram Embedding offload:** at least 51 GB plus
|
||
runtime headroom.
|
||
|
||
The Install block above generates the nightly-wheel and Docker launch flows.
|
||
|
||
## Serving
|
||
|
||
### Recommended FP8 configuration on 4x GB300
|
||
|
||
Use the official
|
||
[`Qwen/Qwen3.8-Flash-Next-FP8`](https://huggingface.co/Qwen/Qwen3.8-Flash-Next-FP8)
|
||
checkpoint tested by this recipe.
|
||
|
||
```bash
|
||
vllm serve Qwen/Qwen3.8-Flash-Next-FP8 \
|
||
--tensor-parallel-size 4 \
|
||
--enable-expert-parallel \
|
||
--gpu-memory-utilization 0.90 \
|
||
--max-num-seqs 256 \
|
||
--enable-prefix-caching \
|
||
--kv-cache-dtype fp8 \
|
||
--attention-config.indexer_kv_dtype fp8 \
|
||
--speculative-config '{"method":"mtp","num_speculative_tokens":3}' \
|
||
--enable-auto-tool-choice \
|
||
--tool-call-parser qwen3_coder \
|
||
--reasoning-parser qwen3
|
||
```
|
||
|
||
TEP4 is the recommended full-tray configuration — we recommend it over plain
|
||
TP4 because expert parallelism is consistently faster for this model. TP4 and
|
||
TEP4 are both validated full-tray configurations, including with MTP3.
|
||
|
||
To serve BF16, use the `Qwen/Qwen3.8-Flash-Next` checkpoint with the same
|
||
arguments.
|
||
|
||
### Single-GPU deployment on GB200/GB300
|
||
|
||
The model fits on a single GPU: the FP8 checkpoint on one GB300 (288 GB
|
||
HBM3e), and the NVFP4 variant on GB200 as well. Single-GPU TP1 suits
|
||
evaluation and development; multi-GPU TEP remains the recommended deployment
|
||
for serving throughput.
|
||
|
||
### Recommended FP8 configuration on 8x H200
|
||
|
||
Use TEP8 with the Triton MoE backend on Hopper:
|
||
|
||
```bash
|
||
vllm serve Qwen/Qwen3.8-Flash-Next-FP8 \
|
||
--tensor-parallel-size 8 \
|
||
--enable-expert-parallel \
|
||
--moe-backend triton \
|
||
--gpu-memory-utilization 0.85 \
|
||
--max-num-seqs 256 \
|
||
--enable-prefix-caching \
|
||
--kv-cache-dtype fp8 \
|
||
--attention-config.indexer_kv_dtype fp8 \
|
||
--speculative-config '{"method":"mtp","num_speculative_tokens":3}' \
|
||
--enable-auto-tool-choice \
|
||
--tool-call-parser qwen3_coder \
|
||
--reasoning-parser qwen3
|
||
```
|
||
|
||
Plain TP8 is incompatible with the FP8 checkpoint; use TEP8.
|
||
|
||
### Recommended FP8 configuration on 8x H100
|
||
|
||
H100's 80GB/GPU is not enough headroom for the 51B N-gram/PLE embedding table
|
||
— it relies on PLE CPU offload, which vLLM enables by default. Use TEP8 with
|
||
the Triton MoE backend; as on other hardware, expert parallelism is faster
|
||
than plain TP, and plain TP8 is incompatible with the FP8 checkpoint's
|
||
128-wide quantization blocks.
|
||
|
||
```bash
|
||
vllm serve Qwen/Qwen3.8-Flash-Next-FP8 \
|
||
--tensor-parallel-size 8 \
|
||
--enable-expert-parallel \
|
||
--moe-backend triton \
|
||
--gpu-memory-utilization 0.85 \
|
||
--max-num-seqs 256 \
|
||
--enable-prefix-caching \
|
||
--kv-cache-dtype fp8 \
|
||
--attention-config.indexer_kv_dtype fp8 \
|
||
--speculative-config '{"method":"mtp","num_speculative_tokens":3}' \
|
||
--enable-auto-tool-choice \
|
||
--tool-call-parser qwen3_coder \
|
||
--reasoning-parser qwen3
|
||
```
|
||
|
||
Benchmark MTP with SPEED-Bench (`--dataset-name speed_bench`), not `random`.
|
||
|
||
### FP8 configuration on a 4x MI355X GPU
|
||
```bash
|
||
export VLLM_ROCM_USE_AITER=1
|
||
export VLLM_ROCM_USE_AITER_MOE=0
|
||
|
||
vllm serve Qwen/Qwen3.8-Flash-Next-FP8 \
|
||
--tensor-parallel-size 4 \
|
||
--max-model-len auto \
|
||
--gpu-memory-utilization 0.9 \
|
||
--speculative-config '{"method":"mtp","num_speculative_tokens":3}'
|
||
```
|
||
|
||
### Full native context
|
||
|
||
The checkpoint advertises a native 262,144-token context, which vLLM uses when
|
||
`--max-model-len` is omitted. Startup and bounded evaluation were validated with
|
||
this limit configured, but a single 262K-token request was not tested.
|
||
|
||
### MTP speculative decoding
|
||
|
||
Add the following option:
|
||
|
||
```bash
|
||
--speculative-config '{"method":"mtp","num_speculative_tokens":3}'
|
||
```
|
||
|
||
### Offload the 51B N-gram embedding table
|
||
|
||
Builds containing N-gram Embedding offload can keep the N-gram lookup
|
||
memory in host RAM and asynchronously prefetch the required rows. The initial
|
||
implementation supports both ModelRunner V1 and V2. PLE CPU offload is required
|
||
for DEP, and also required for TP/TEP on 80GB-class GPUs (H100) where the
|
||
embedding table alone exceeds available headroom — optional only on GPUs with
|
||
more VRAM per rank (H200 and above). vLLM enables it by default; set it
|
||
explicitly with `--engram-config '{"cpu_offload":true}'`.
|
||
|
||
```bash
|
||
vllm serve Qwen/Qwen3.8-Flash-Next-FP8 \
|
||
--tensor-parallel-size 4 \
|
||
--enable-expert-parallel \
|
||
--max-model-len 262144 \
|
||
--enable-auto-tool-choice \
|
||
--tool-call-parser qwen3_coder \
|
||
--reasoning-parser qwen3
|
||
```
|
||
|
||
### Extend to one million tokens with YaRN
|
||
|
||
The checkpoint is native at 262K. For a one-million-token workload, turn on the
|
||
**Long Context** feature above: it applies the model card's YaRN RoPE
|
||
configuration under `text_config` (where this checkpoint carries
|
||
`rope_parameters`) and raises `--max-model-len` to 1,000,000. Evaluate
|
||
shorter-context quality before making it the default — static YaRN holds the
|
||
scaling factor constant regardless of input length.
|
||
|
||
## Verifying
|
||
|
||
```python
|
||
from openai import OpenAI
|
||
|
||
client = OpenAI(api_key="EMPTY", base_url="http://localhost:8000/v1")
|
||
response = client.chat.completions.create(
|
||
model="Qwen/Qwen3.8-Flash-Next-FP8",
|
||
messages=[{
|
||
"role": "user",
|
||
"content": "Explain how Gated DeltaNet and Qwen Sparse Attention complement each other.",
|
||
}],
|
||
max_tokens=8196,
|
||
)
|
||
print(response.choices[0].message.content)
|
||
```
|
||
|
||
A correct response should explain that GDN maintains a compact recurrent summary,
|
||
while QSA selectively retrieves important regions from the full token history.
|
||
|
||
## Troubleshooting
|
||
|
||
- **FP8 on eight GPUs:** use TEP8; plain TP8 is incompatible with this
|
||
checkpoint. TP4 is also supported if GPU memory capacity allows.
|
||
- **Mamba-cache capacity error at startup:** keep `--max-num-seqs 256`.
|
||
- **Runtime OOM on large multimodal batches:** keep `--gpu-memory-utilization 0.90`.
|
||
- **Out of memory while loading:** enable N-gram Embedding CPU offload, increase
|
||
TP size, or reduce `--max-model-len` to reserve less KV cache. On 4x H100
|
||
80GB, plain TP4 OOMs at engine startup without PLE CPU offload — keep the
|
||
default `--engram-config '{"cpu_offload":true}'`.
|
||
- **XPU/TPU startup error:** these platforms are not supported by the initial
|
||
implementation.
|
||
- **Pipeline-parallel startup error:** N-gram Embedding does not initially support
|
||
pipeline parallelism; use single-node TP or TEP instead.
|
||
- **DEP startup failure:** DEP requires PLE CPU offload (on by default); DEP
|
||
without it is not supported.
|
||
- **MTP memory pressure:** reduce `num_speculative_tokens` below 3.
|
||
- **PLE:** a network layer that injects N-gram Embeddings into the main model.
|
||
|
||
## KV cache offloading
|
||
|
||
Offloading is available upstream since vLLM v0.30.0 ([vllm-project/vllm#57145](https://github.com/vllm-project/vllm/pull/57145)).
|
||
Validation used [Qwen/Qwen3.8-Flash-Next-FP8](https://huggingface.co/Qwen/Qwen3.8-Flash-Next-FP8) on 4×H200 (TP4),
|
||
vLLM commit `d1b4028d7e`, the Python frontend, `--enforce-eager`,
|
||
`--max-model-len 16384`, `--max-num-seqs 16` and `--max-num-batched-tokens 8192`.
|
||
Qwen validation used the FP8 variant with `--limit-mm-per-prompt '{"image":0,"video":0}'`.
|
||
Validation used `TieringOffloadingSpec`: 8 GiB total CPU tier, `blocks_per_chunk: 1`,
|
||
a persistent filesystem `root_dir`, eight read/write threads and `PYTHONHASHSEED=0`.
|
||
Three CPU-tier reloads and three filesystem reloads after restarting the server passed.
|
||
Reloaded responses matched the cold output, including the reasoning field.
|
||
The five-shot GSM8K diagnostic scored 31/32; this is a small sample, not the full benchmark.
|
||
These results cover this text-only configuration, not the builder's larger tier/chunk defaults or other variants.
|
||
|
||
## References
|
||
|
||
- [Qwen3.8-Flash-Next model card](https://huggingface.co/Qwen/Qwen3.8-Flash-Next)
|
||
- [Qwen3.8-Flash-Next FP8 checkpoint](https://huggingface.co/Qwen/Qwen3.8-Flash-Next-FP8)
|
||
- [Qwen3.8-Flash-Next: A New Architecture, Towards Ultimate Cost-Efficiency](https://qwen.ai/blog?id=qwen3.8-flash-next)
|
||
- [Qwen3-Next recipe](../Qwen3-Next-80B-A3B-Instruct)
|