Enable DeepSeek V4 and GLM-5.1 on SM120 (#43477)

Signed-off-by: Zihua Wu <13583761+lucifer1004@users.noreply.github.com>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
This commit is contained in:
Gabriel Wu
2026-06-22 11:54:14 -07:00
committed by GitHub
co-authored by Claude Opus 4.8 Yongye Zhu
parent 3ce15fd574
commit 44d95069e9
37 changed files with 2347 additions and 476 deletions
+130
View File
@@ -2,12 +2,14 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable
from contextlib import AbstractContextManager, nullcontext
from typing import Any
import numpy as np
import torch
from vllm import PoolingParams, SamplingParams
from vllm.logger import init_logger
from vllm.utils.math_utils import cdiv
from vllm.v1.core.sched.output import (
CachedRequestData,
@@ -18,6 +20,134 @@ from vllm.v1.core.sched.output import (
from vllm.v1.request import Request
from vllm.v1.worker.gpu.model_runner import GPUModelRunner
logger = init_logger(__name__)
def run_mixed_prefill_decode_warmup(
model_runner: GPUModelRunner,
worker_execute_model: Callable[[SchedulerOutput], Any],
worker_sample_tokens: Callable[[GrammarOutput | None], Any],
num_tokens: int,
*,
mixed_step_context: AbstractContextManager[object] | None = None,
req_id_prefix: str = "_v2_mixed_warmup",
) -> bool:
"""Run a V2 mixed prefill+decode step through normal scheduler inputs."""
if model_runner.is_pooling_model or num_tokens < 3:
return False
decode_req_id = f"{req_id_prefix}_decode_"
prefill_req_id = f"{req_id_prefix}_prefill_"
decode_prompt_len = 2
decode_scheduled_tokens = 1
prefill_len = num_tokens - decode_scheduled_tokens
decode_token_ids = list(range(decode_prompt_len))
prefill_token_ids = list(range(prefill_len))
kv_cache_groups = model_runner.kv_cache_config.kv_cache_groups
num_kv_cache_groups = len(kv_cache_groups)
group_block_sizes = [g.kv_cache_spec.block_size for g in kv_cache_groups]
decode_prefill_block_counts = [
cdiv(decode_prompt_len, block_size) for block_size in group_block_sizes
]
decode_block_counts = [
cdiv(decode_prompt_len + decode_scheduled_tokens, block_size)
for block_size in group_block_sizes
]
decode_block_deltas = [
decode - prefill
for decode, prefill in zip(decode_block_counts, decode_prefill_block_counts)
]
prefill_block_counts = [
cdiv(prefill_len, block_size) for block_size in group_block_sizes
]
required_blocks = sum(decode_block_counts) + sum(prefill_block_counts)
if model_runner.kv_cache_config.num_blocks <= required_blocks:
logger.warning(
"Skipping V2 mixed prefill+decode warmup because only %d KV blocks "
"are available for %d required warmup blocks.",
model_runner.kv_cache_config.num_blocks,
required_blocks,
)
return False
next_block_id = 1
def _alloc_blocks(num_blocks: int) -> list[int]:
nonlocal next_block_id
block_ids = list(range(next_block_id, next_block_id + num_blocks))
next_block_id += num_blocks
return block_ids
sampling_params = SamplingParams(max_tokens=2, temperature=0.0)
decode_prefill_output = SchedulerOutput.make_empty()
decode_prefill_output.scheduled_new_reqs = [
NewRequestData(
req_id=decode_req_id,
prompt_token_ids=decode_token_ids,
mm_features=[],
sampling_params=sampling_params,
pooling_params=None,
block_ids=tuple(_alloc_blocks(n) for n in decode_prefill_block_counts),
num_computed_tokens=0,
lora_request=None,
prefill_token_ids=decode_token_ids,
),
]
decode_prefill_output.num_scheduled_tokens = {
decode_req_id: decode_prompt_len,
}
decode_prefill_output.total_num_scheduled_tokens = decode_prompt_len
decode_prefill_output.num_common_prefix_blocks = [0] * num_kv_cache_groups
decode_new_blocks = tuple(_alloc_blocks(n) for n in decode_block_deltas)
cached_decode_req = CachedRequestData.make_empty()
cached_decode_req.req_ids = [decode_req_id]
cached_decode_req.num_computed_tokens = [decode_prompt_len]
cached_decode_req.num_output_tokens = [1]
cached_decode_req.new_block_ids = [
decode_new_blocks if any(decode_block_deltas) else None
]
mixed_output = SchedulerOutput.make_empty()
mixed_output.scheduled_cached_reqs = cached_decode_req
mixed_output.scheduled_new_reqs = [
NewRequestData(
req_id=prefill_req_id,
prompt_token_ids=prefill_token_ids,
mm_features=[],
sampling_params=sampling_params,
pooling_params=None,
block_ids=tuple(_alloc_blocks(n) for n in prefill_block_counts),
num_computed_tokens=0,
lora_request=None,
prefill_token_ids=prefill_token_ids,
),
]
mixed_output.num_scheduled_tokens = {
decode_req_id: decode_scheduled_tokens,
prefill_req_id: prefill_len,
}
mixed_output.total_num_scheduled_tokens = num_tokens
mixed_output.num_common_prefix_blocks = [0] * num_kv_cache_groups
cleanup_output = SchedulerOutput.make_empty()
cleanup_output.finished_req_ids = {decode_req_id, prefill_req_id}
context = mixed_step_context or nullcontext()
model_runner.kv_connector.set_disabled(True)
try:
worker_execute_model(decode_prefill_output)
worker_sample_tokens(None)
with context:
worker_execute_model(mixed_output)
worker_sample_tokens(None)
worker_execute_model(cleanup_output)
finally:
model_runner.kv_connector.set_disabled(False)
return True
@torch.inference_mode()
def warmup_kernels(