From e30f39c4f13171b8d8047f1c5a4f071d63ec195e Mon Sep 17 00:00:00 2001 From: Louie Tsai Date: Thu, 14 May 2026 22:14:03 -0700 Subject: [PATCH] Update Intel Xeon model list and vLLM Benchmark Suite BKMs (#42607) Signed-off-by: louie-tsai --- .../tests/serving-tests-cpu-text.json | 339 +++++++----------- docs/models/hardware_supported_models/cpu.md | 56 ++- 2 files changed, 177 insertions(+), 218 deletions(-) diff --git a/.buildkite/performance-benchmarks/tests/serving-tests-cpu-text.json b/.buildkite/performance-benchmarks/tests/serving-tests-cpu-text.json index 6c4591f05b..34c2cc82d3 100644 --- a/.buildkite/performance-benchmarks/tests/serving-tests-cpu-text.json +++ b/.buildkite/performance-benchmarks/tests/serving-tests-cpu-text.json @@ -31,30 +31,9 @@ } }, "tests": [ - { - "test_name": "serving_llama8B_tp1_sharegpt", - "server_parameters": { - "tensor_parallel_size": 1 - }, - "client_parameters": { - "dataset_name": "sharegpt", - "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json" - } - }, - { - "test_name": "serving_llama8B_tp2_sharegpt", - "server_parameters": { - "tensor_parallel_size": 2 - }, - "client_parameters": { - "dataset_name": "sharegpt", - "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json" - } - }, { "test_name": "serving_llama8B_tp1_random_128_128", "server_parameters": { - "tensor_parallel_size": 1 }, "client_parameters": { "dataset_name": "random", @@ -62,158 +41,10 @@ "random-output-len": 128 } }, - { - "test_name": "serving_llama8B_tp2_random_128_128", - "server_parameters": { - "tensor_parallel_size": 2 - }, - "client_parameters": { - "dataset_name": "random", - "random-input-len": 128, - "random-output-len": 128 - } - }, - { - "test_name": "serving_llama8B_tp4_random_128_128", - "server_parameters": { - "tensor_parallel_size": 4 - }, - "client_parameters": { - "dataset_name": "random", - "random-input-len": 128, - "random-output-len": 128 - } - }, - { - "test_name": "serving_llama8B_tp1_random_128_2048", - "server_parameters": { - "tensor_parallel_size": 1 - }, - "client_parameters": { - "dataset_name": "random", - "random-input-len": 128, - "random-output-len": 2048 - } - }, - { - "test_name": "serving_llama8B_tp2_random_128_2048", - "server_parameters": { - "tensor_parallel_size": 2 - }, - "client_parameters": { - "dataset_name": "random", - "random-input-len": 128, - "random-output-len": 2048 - } - }, - { - "test_name": "serving_llama8B_tp4_random_128_2048", - "server_parameters": { - "tensor_parallel_size": 4 - }, - "client_parameters": { - "dataset_name": "random", - "random-input-len": 128, - "random-output-len": 2048 - } - }, - { - "test_name": "serving_llama8B_tp1_random_2048_128", - "server_parameters": { - "tensor_parallel_size": 1 - }, - "client_parameters": { - "dataset_name": "random", - "random-input-len": 2048, - "random-output-len": 128 - } - }, - { - "test_name": "serving_llama8B_tp2_random_2048_128", - "server_parameters": { - "tensor_parallel_size": 2 - }, - "client_parameters": { - "dataset_name": "random", - "random-input-len": 2048, - "random-output-len": 128 - } - }, - { - "test_name": "serving_llama8B_tp4_random_2048_128", - "server_parameters": { - "tensor_parallel_size": 4 - }, - "client_parameters": { - "dataset_name": "random", - "random-input-len": 2048, - "random-output-len": 128 - } - }, - { - "test_name": "serving_llama8B_tp1_random_2048_2048", - "server_parameters": { - "tensor_parallel_size": 1 - }, - "client_parameters": { - "dataset_name": "random", - "random-input-len": 2048, - "random-output-len": 2048 - } - }, - { - "test_name": "serving_llama8B_tp2_random_2048_2048", - "server_parameters": { - "tensor_parallel_size": 2 - }, - "client_parameters": { - "dataset_name": "random", - "random-input-len": 2048, - "random-output-len": 2048 - } - }, - { - "test_name": "serving_llama8B_tp4_random_2048_2048", - "server_parameters": { - "tensor_parallel_size": 4 - }, - "client_parameters": { - "dataset_name": "random", - "random-input-len": 2048, - "random-output-len": 2048 - } - }, { "test_name": "serving_llama8B_int4_tp1_random_128_128", "server_parameters": { - "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4", - "tensor_parallel_size": 1 - }, - "client_parameters": { - "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4", - "dataset_name": "random", - "random-input-len": 128, - "random-output-len": 128 - } - }, - { - "test_name": "serving_llama8B_int4_tp2_random_128_128", - "server_parameters": { - "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4", - "tensor_parallel_size": 2 - }, - "client_parameters": { - "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4", - "dataset_name": "random", - "random-input-len": 128, - "random-output-len": 128 - } - }, - { - "test_name": "serving_llama8B_int4_tp4_random_128_128", - "server_parameters": { - "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4", - "tensor_parallel_size": 4 + "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4" }, "client_parameters": { "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4", @@ -225,8 +56,7 @@ { "test_name": "serving_llama8B_int8_tp1_random_128_128", "server_parameters": { - "model": "RedHatAI/Meta-Llama-3.1-8B-Instruct-quantized.w8a8", - "tensor_parallel_size": 1 + "model": "RedHatAI/Meta-Llama-3.1-8B-Instruct-quantized.w8a8" }, "client_parameters": { "model": "RedHatAI/Meta-Llama-3.1-8B-Instruct-quantized.w8a8", @@ -236,26 +66,12 @@ } }, { - "test_name": "serving_llama8B_int8_tp2_random_128_128", + "test_name": "serving_llama1B_tp1_random_128_128", "server_parameters": { - "model": "RedHatAI/Meta-Llama-3.1-8B-Instruct-quantized.w8a8", - "tensor_parallel_size": 2 + "model": "meta-llama/Llama-3.2-1B" }, "client_parameters": { - "model": "RedHatAI/Meta-Llama-3.1-8B-Instruct-quantized.w8a8", - "dataset_name": "random", - "random-input-len": 128, - "random-output-len": 128 - } - }, - { - "test_name": "serving_llama8B_int8_tp4_random_128_128", - "server_parameters": { - "model": "RedHatAI/Meta-Llama-3.1-8B-Instruct-quantized.w8a8", - "tensor_parallel_size": 4 - }, - "client_parameters": { - "model": "RedHatAI/Meta-Llama-3.1-8B-Instruct-quantized.w8a8", + "model": "meta-llama/Llama-3.2-1B", "dataset_name": "random", "random-input-len": 128, "random-output-len": 128 @@ -264,8 +80,7 @@ { "test_name": "serving_llama3B_tp1_random_128_128", "server_parameters": { - "model": "meta-llama/Llama-3.2-3B-Instruct", - "tensor_parallel_size": 1 + "model": "meta-llama/Llama-3.2-3B-Instruct" }, "client_parameters": { "model": "meta-llama/Llama-3.2-3B-Instruct", @@ -274,11 +89,22 @@ "random-output-len": 128 } }, + { + "test_name": "serving_llama70B_tp1_random_128_128", + "server_parameters": { + "model": "meta-llama/Llama-3.3-70B-Instruct" + }, + "client_parameters": { + "model": "meta-llama/Llama-3.3-70B-Instruct", + "dataset_name": "random", + "random-input-len": 128, + "random-output-len": 128 + } + }, { "test_name": "serving_granite2B_tp1_random_128_128", "server_parameters": { - "model": "ibm-granite/granite-3.2-2b-instruct", - "tensor_parallel_size": 1 + "model": "ibm-granite/granite-3.2-2b-instruct" }, "client_parameters": { "model": "ibm-granite/granite-3.2-2b-instruct", @@ -290,8 +116,7 @@ { "test_name": "serving_qwen1.7B_tp1_random_128_128", "server_parameters": { - "model": "Qwen/Qwen3-1.7B", - "tensor_parallel_size": 1 + "model": "Qwen/Qwen3-1.7B" }, "client_parameters": { "model": "Qwen/Qwen3-1.7B", @@ -303,8 +128,7 @@ { "test_name": "serving_qwen4B_tp1_random_128_128", "server_parameters": { - "model": "Qwen/Qwen3-4B", - "tensor_parallel_size": 1 + "model": "Qwen/Qwen3-4B" }, "client_parameters": { "model": "Qwen/Qwen3-4B", @@ -316,8 +140,7 @@ { "test_name": "serving_qwen8B_tp1_random_128_128", "server_parameters": { - "model": "Qwen/Qwen3-8B", - "tensor_parallel_size": 1 + "model": "Qwen/Qwen3-8B" }, "client_parameters": { "model": "Qwen/Qwen3-8B", @@ -326,11 +149,34 @@ "random-output-len": 128 } }, + { + "test_name": "serving_qwen14B_tp1_random_128_128", + "server_parameters": { + "model": "Qwen/Qwen3-14B" + }, + "client_parameters": { + "model": "Qwen/Qwen3-14B", + "dataset_name": "random", + "random-input-len": 128, + "random-output-len": 128 + } + }, + { + "test_name": "serving_qwen30B_tp1_random_128_128", + "server_parameters": { + "model": "Qwen/Qwen3-30B-A3B" + }, + "client_parameters": { + "model": "Qwen/Qwen3-30B-A3B", + "dataset_name": "random", + "random-input-len": 128, + "random-output-len": 128 + } + }, { "test_name": "serving_glm9B_tp1_random_128_128", "server_parameters": { - "model": "zai-org/glm-4-9b-hf", - "tensor_parallel_size": 1 + "model": "zai-org/glm-4-9b-hf" }, "client_parameters": { "model": "zai-org/glm-4-9b-hf", @@ -342,8 +188,7 @@ { "test_name": "serving_gemma7B_tp1_random_128_128", "server_parameters": { - "model": "google/gemma-7b", - "tensor_parallel_size": 1 + "model": "google/gemma-7b" }, "client_parameters": { "model": "google/gemma-7b", @@ -351,6 +196,94 @@ "random-input-len": 128, "random-output-len": 128 } + }, + { + "test_name": "serving_gemma3-4b_tp1_random_128_128", + "server_environment_variables": { + "VLLM_CPU_SGL_KERNEL": 0 + }, + "server_parameters": { + "model": "google/gemma-3-4b-it" + }, + "client_parameters": { + "model": "google/gemma-3-4b-it", + "dataset_name": "random", + "random-input-len": 128, + "random-output-len": 128 + } + }, + { + "test_name": "serving_gemma3-12b_tp1_random_128_128", + "server_environment_variables": { + "VLLM_CPU_SGL_KERNEL": 0 + }, + "server_parameters": { + "model": "google/gemma-3-12b-it" + }, + "client_parameters": { + "model": "google/gemma-3-12b-it", + "dataset_name": "random", + "random-input-len": 128, + "random-output-len": 128 + } + }, + { + "test_name": "serving_gemma4-4b_tp1_random_128_128", + "server_environment_variables": { + "VLLM_CPU_SGL_KERNEL": 0 + }, + "server_parameters": { + "model": "google/gemma-4-E4B-it" + }, + "client_parameters": { + "model": "google/gemma-4-E4B-it", + "dataset_name": "random", + "random-input-len": 128, + "random-output-len": 128 + } + }, + { + "test_name": "serving_gemma4-2b_tp1_random_128_128", + "server_environment_variables": { + "VLLM_CPU_SGL_KERNEL": 0 + }, + "server_parameters": { + "model": "google/gemma-4-E2B-it" + }, + "client_parameters": { + "model": "google/gemma-4-E2B-it", + "dataset_name": "random", + "random-input-len": 128, + "random-output-len": 128 + } + }, + { + "test_name": "serving_gemma4-26b_tp1_random_128_128", + "server_environment_variables": { + "VLLM_CPU_SGL_KERNEL": 0, + "VLLM_CPU_ATTN_SPLIT_KV": 0 + }, + "server_parameters": { + "model": "google/gemma-4-26B-A4B-it" + }, + "client_parameters": { + "model": "google/gemma-4-26B-A4B-it", + "dataset_name": "random", + "random-input-len": 128, + "random-output-len": 128 + } + }, + { + "test_name": "serving_phi4_tp1_random_128_128", + "server_parameters": { + "model": "microsoft/Phi-4-reasoning" + }, + "client_parameters": { + "model": "microsoft/Phi-4-reasoning", + "dataset_name": "random", + "random-input-len": 128, + "random-output-len": 128 + } } ] } diff --git a/docs/models/hardware_supported_models/cpu.md b/docs/models/hardware_supported_models/cpu.md index 361310f18c..9c6dd9feb7 100644 --- a/docs/models/hardware_supported_models/cpu.md +++ b/docs/models/hardware_supported_models/cpu.md @@ -11,24 +11,50 @@ ### Text-only Language Models -| Model | Architecture | Supported | +| Model | Architecture | Supported | | ------------------------------------ | ---------------------------------------- | --------- | -| meta-llama/Llama-3.1-8B-Instruct | LlamaForCausalLM | ✅ | -| meta-llama/Llama-3.2-3B-Instruct | LlamaForCausalLM | ✅ | -| ibm-granite/granite-3.2-2b-instruct | GraniteForCausalLM | ✅ | -| Qwen/Qwen3-1.7B | Qwen3ForCausalLM | ✅ | -| Qwen/Qwen3-4B | Qwen3ForCausalLM | ✅ | -| Qwen/Qwen3-8B | Qwen3ForCausalLM | ✅ | -| zai-org/glm-4-9b-hf | GLMForCausalLM | ✅ | -| google/gemma-7b | GemmaForCausalLM | ✅ | +| unsloth/gpt-oss-20b | GptOssForCausalLM | ✅ | +| meta-llama/Llama-3.1-8B-Instruct | LlamaForCausalLM | ✅ | +| meta-llama/Llama-3.2-1B | LlamaForCausalLM | ✅ | +| meta-llama/Llama-3.2-3B-Instruct | LlamaForCausalLM | ✅ | +| meta-llama/Llama-3.3-70B-Instruct | LlamaForCausalLM | ✅ | +| RedHatAI/Meta-Llama-3.1-8B-quantized.w8a8 | LlamaForCausalLM | ✅ | +| RedHatAI/Meta-Llama-3.1-8B-Instruct-quantized.w8a8 | LlamaForCausalLM | ✅ | +| RedHatAI/Llama-3.2-1B-Instruct-quantized.w8a8 | LlamaForCausalLM | ✅ | +| RedHatAI/Llama-3.2-3B-Instruct-quantized.w8a8 | LlamaForCausalLM | ✅ | +| RedHatAI/DeepSeek-R1-Distill-Llama-70B-quantized.w8a8 | LlamaForCausalLM | ✅ | +| hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4 | LlamaForCausalLM | ✅ | +| AMead10/Llama-3.2-1B-Instruct-AWQ | LlamaForCausalLM | ✅ | +| AMead10/Llama-3.2-3B-Instruct-AWQ | LlamaForCausalLM | ✅ | +| TheBloke/TinyLlama-1.1B-Chat-v1.0-AWQ | LlamaForCausalLM | ✅ | +| TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ | LlamaForCausalLM | ✅ | +| ibm-granite/granite-3.2-2b-instruct | GraniteForCausalLM | ✅ | +| Qwen/Qwen3-1.7B | Qwen3ForCausalLM | ✅ | +| Qwen/Qwen3-4B | Qwen3ForCausalLM | ✅ | +| Qwen/Qwen3-8B | Qwen3ForCausalLM | ✅ | +| Qwen/Qwen3-14B | Qwen3ForCausalLM | ✅ | +| Qwen/Qwen3-14B-AWQ | Qwen3ForCausalLM | ✅ | +| Qwen/Qwen3-30B-A3B | Qwen3MoeForCausalLM | ✅ | +| Qwen/QwQ-32B-AWQ | Qwen2ForCausalLM | ✅ | +| Qwen/Qwen1.5-0.5B-Chat-GPTQ-Int4 | Qwen2ForCausalLM | ✅ | +| RedHatAI/QwQ-32B-quantized.w8a8 | Qwen2ForCausalLM | ✅ | +| zai-org/glm-4-9b-hf | GLMForCausalLM | ✅ | +| google/gemma-7b | GemmaForCausalLM | ✅ | +| microsoft/Phi-4-reasoning | Phi3ForCausalLM | ✅ | +| TheBloke/Mistral-7B-Instruct-v0.2-AWQ | MistralForCausalLM | ✅ | ### Multimodal Language Models -| Model | Architecture | Supported | +| Model | Architecture | Supported | | ------------------------------------ | ---------------------------------------- | --------- | -| Qwen/Qwen2.5-VL-7B-Instruct | Qwen2VLForConditionalGeneration | ✅ | -| openai/whisper-large-v3 | WhisperForConditionalGeneration | ✅ | +| meta-llama/Llama-4-Scout-17B-16E-Instruct | Llama4ForConditionalGeneration | ✅ | +| google/gemma-3-4b-it | Gemma3ForConditionalGeneration | ✅ | +| google/gemma-3-12b-it | Gemma3ForConditionalGeneration | ✅ | +| google/gemma-4-E4B-it | Gemma4ForConditionalGeneration | ✅ | +| google/gemma-4-E2B-it | Gemma4ForConditionalGeneration | ✅ | +| google/gemma-4-26B-A4B-it | Gemma4ForConditionalGeneration | ✅ | +| microsoft/Phi-4-multimodal-instruct | Phi4MMForCausalLM | ✅ | +| Qwen/Qwen2.5-VL-7B-Instruct | Qwen2VLForConditionalGeneration | ✅ | +| openai/whisper-large-v3 | WhisperForConditionalGeneration | ✅ | -✅ Runs and optimized. -🟨 Runs and correct but not optimized to green yet. -❌ Does not pass accuracy test or does not run. +✅ Runs and optimized.