iHore/vLLM
8
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[Doc][Recipe] Fix for redhat fp8 xeon support issue (#62245)

Browse Source
This commit is contained in:
Louie Tsai
2026-10-18 16:46:56 +08:00
committed by GitHub
parent 83c9a547bf
commit 1b2a666391
5 changed files with 11 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
recipes/models/Qwen/Qwen2.5-VL-7B-Instruct.yaml
+1
View File
@@ -60,6 +60,7 @@ variants:
precision: fp8
vram_minimum_gb: 9
description: "RedHatAI FP8 (E4M3) with dynamic per-token activation quantization"
supported_hardware: [trillium]
extra_args:
- "--kv-cache-dtype"
- "fp8"
recipes/models/Qwen/Qwen3-4B.yaml
+1
View File
@@ -66,6 +66,7 @@ variants:
label: "FP8 Dynamic (RedHat)"
vram_minimum_gb: 5
description: "RedHatAI FP8 (E4M3) with dynamic per-token activation quantization"
supported_hardware: [trillium]
extra_args:
- "--kv-cache-dtype"
- "fp8"
recipes/models/meta-llama/Llama-3.1-8B-Instruct.yaml
+3
View File
@@ -88,6 +88,7 @@ variants:
label: "FP8 Dynamic (RedHat)"
vram_minimum_gb: 10
description: "RedHatAI FP8 (E4M3) with dynamic per-token activation quantization"
supported_hardware: [h100, h200]
extra_args:
- "--kv-cache-dtype"
- "fp8"
@@ -98,6 +99,7 @@ variants:
label: "FP8 (RedHat)"
vram_minimum_gb: 10
description: "RedHatAI FP8 quantization"
supported_hardware: [h100, h200]
extra_args:
- "--kv-cache-dtype"
- "fp8"
@@ -108,6 +110,7 @@ variants:
label: "FP8 Block (RedHat)"
vram_minimum_gb: 10
description: "RedHatAI FP8 block quantization"
supported_hardware: [h100, h200, xeon6]
extra_args:
- "--kv-cache-dtype"
- "fp8"
recipes/models/meta-llama/Llama-3.3-70B-Instruct.yaml
+3
View File
@@ -75,6 +75,7 @@ variants:
label: "FP8 (RedHat)"
vram_minimum_gb: 84
description: "RedHatAI FP8 (E4M3) with dynamic per-token activation quantization"
supported_hardware: [h100, h200, b200, gb200]
extra_args:
- "--kv-cache-dtype"
- "fp8"
@@ -95,6 +96,7 @@ variants:
label: "FP8 Block (RedHat)"
vram_minimum_gb: 84
description: "RedHatAI FP8 block quantization"
supported_hardware: [h100, h200, b200, gb200, xeon6]
extra_args:
- "--kv-cache-dtype"
- "fp8"
@@ -105,6 +107,7 @@ variants:
label: "NVFP4 (RedHat)"
vram_minimum_gb: 46
description: "RedHatAI NVFP4 quantization"
supported_hardware: [b200, gb200]
compatible_strategies:
- single_node_tp
recipes/models/meta-llama/Llama-4-Scout-17B-16E-Instruct.yaml
+3
View File
@@ -67,6 +67,7 @@ variants:
label: "FP8 (RedHat)"
vram_minimum_gb: 131
description: "RedHatAI FP8 (E4M3) with dynamic per-token activation quantization"
supported_hardware: [h100, b200, gb200]
extra_args:
- "--kv-cache-dtype"
- "fp8"
@@ -77,6 +78,7 @@ variants:
label: "NVFP4 (RedHat)"
vram_minimum_gb: 65
description: "RedHatAI compressed-tensors NVFP4 quantization for Blackwell GPUs"
supported_hardware: [b200, gb200]
extra_args:
- "--kv-cache-dtype"
- "fp8"
@@ -89,6 +91,7 @@ variants:
label: "FP8 Block (RedHat)"
vram_minimum_gb: 131
description: "RedHatAI FP8 block quantization"
supported_hardware: [h100, b200, gb200]
extra_args:
- "--kv-cache-dtype"
- "fp8"