Enable DeepSeek V4 and GLM-5.1 on SM120 (#43477)
Signed-off-by: Zihua Wu <13583761+lucifer1004@users.noreply.github.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
Yongye Zhu
parent
3ce15fd574
commit
44d95069e9
@@ -2,12 +2,14 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections.abc import Callable
|
||||
from contextlib import AbstractContextManager, nullcontext
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from vllm import PoolingParams, SamplingParams
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.v1.core.sched.output import (
|
||||
CachedRequestData,
|
||||
@@ -18,6 +20,134 @@ from vllm.v1.core.sched.output import (
|
||||
from vllm.v1.request import Request
|
||||
from vllm.v1.worker.gpu.model_runner import GPUModelRunner
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def run_mixed_prefill_decode_warmup(
|
||||
model_runner: GPUModelRunner,
|
||||
worker_execute_model: Callable[[SchedulerOutput], Any],
|
||||
worker_sample_tokens: Callable[[GrammarOutput | None], Any],
|
||||
num_tokens: int,
|
||||
*,
|
||||
mixed_step_context: AbstractContextManager[object] | None = None,
|
||||
req_id_prefix: str = "_v2_mixed_warmup",
|
||||
) -> bool:
|
||||
"""Run a V2 mixed prefill+decode step through normal scheduler inputs."""
|
||||
if model_runner.is_pooling_model or num_tokens < 3:
|
||||
return False
|
||||
|
||||
decode_req_id = f"{req_id_prefix}_decode_"
|
||||
prefill_req_id = f"{req_id_prefix}_prefill_"
|
||||
decode_prompt_len = 2
|
||||
decode_scheduled_tokens = 1
|
||||
prefill_len = num_tokens - decode_scheduled_tokens
|
||||
decode_token_ids = list(range(decode_prompt_len))
|
||||
prefill_token_ids = list(range(prefill_len))
|
||||
|
||||
kv_cache_groups = model_runner.kv_cache_config.kv_cache_groups
|
||||
num_kv_cache_groups = len(kv_cache_groups)
|
||||
group_block_sizes = [g.kv_cache_spec.block_size for g in kv_cache_groups]
|
||||
decode_prefill_block_counts = [
|
||||
cdiv(decode_prompt_len, block_size) for block_size in group_block_sizes
|
||||
]
|
||||
decode_block_counts = [
|
||||
cdiv(decode_prompt_len + decode_scheduled_tokens, block_size)
|
||||
for block_size in group_block_sizes
|
||||
]
|
||||
decode_block_deltas = [
|
||||
decode - prefill
|
||||
for decode, prefill in zip(decode_block_counts, decode_prefill_block_counts)
|
||||
]
|
||||
prefill_block_counts = [
|
||||
cdiv(prefill_len, block_size) for block_size in group_block_sizes
|
||||
]
|
||||
required_blocks = sum(decode_block_counts) + sum(prefill_block_counts)
|
||||
if model_runner.kv_cache_config.num_blocks <= required_blocks:
|
||||
logger.warning(
|
||||
"Skipping V2 mixed prefill+decode warmup because only %d KV blocks "
|
||||
"are available for %d required warmup blocks.",
|
||||
model_runner.kv_cache_config.num_blocks,
|
||||
required_blocks,
|
||||
)
|
||||
return False
|
||||
|
||||
next_block_id = 1
|
||||
|
||||
def _alloc_blocks(num_blocks: int) -> list[int]:
|
||||
nonlocal next_block_id
|
||||
block_ids = list(range(next_block_id, next_block_id + num_blocks))
|
||||
next_block_id += num_blocks
|
||||
return block_ids
|
||||
|
||||
sampling_params = SamplingParams(max_tokens=2, temperature=0.0)
|
||||
|
||||
decode_prefill_output = SchedulerOutput.make_empty()
|
||||
decode_prefill_output.scheduled_new_reqs = [
|
||||
NewRequestData(
|
||||
req_id=decode_req_id,
|
||||
prompt_token_ids=decode_token_ids,
|
||||
mm_features=[],
|
||||
sampling_params=sampling_params,
|
||||
pooling_params=None,
|
||||
block_ids=tuple(_alloc_blocks(n) for n in decode_prefill_block_counts),
|
||||
num_computed_tokens=0,
|
||||
lora_request=None,
|
||||
prefill_token_ids=decode_token_ids,
|
||||
),
|
||||
]
|
||||
decode_prefill_output.num_scheduled_tokens = {
|
||||
decode_req_id: decode_prompt_len,
|
||||
}
|
||||
decode_prefill_output.total_num_scheduled_tokens = decode_prompt_len
|
||||
decode_prefill_output.num_common_prefix_blocks = [0] * num_kv_cache_groups
|
||||
|
||||
decode_new_blocks = tuple(_alloc_blocks(n) for n in decode_block_deltas)
|
||||
cached_decode_req = CachedRequestData.make_empty()
|
||||
cached_decode_req.req_ids = [decode_req_id]
|
||||
cached_decode_req.num_computed_tokens = [decode_prompt_len]
|
||||
cached_decode_req.num_output_tokens = [1]
|
||||
cached_decode_req.new_block_ids = [
|
||||
decode_new_blocks if any(decode_block_deltas) else None
|
||||
]
|
||||
|
||||
mixed_output = SchedulerOutput.make_empty()
|
||||
mixed_output.scheduled_cached_reqs = cached_decode_req
|
||||
mixed_output.scheduled_new_reqs = [
|
||||
NewRequestData(
|
||||
req_id=prefill_req_id,
|
||||
prompt_token_ids=prefill_token_ids,
|
||||
mm_features=[],
|
||||
sampling_params=sampling_params,
|
||||
pooling_params=None,
|
||||
block_ids=tuple(_alloc_blocks(n) for n in prefill_block_counts),
|
||||
num_computed_tokens=0,
|
||||
lora_request=None,
|
||||
prefill_token_ids=prefill_token_ids,
|
||||
),
|
||||
]
|
||||
mixed_output.num_scheduled_tokens = {
|
||||
decode_req_id: decode_scheduled_tokens,
|
||||
prefill_req_id: prefill_len,
|
||||
}
|
||||
mixed_output.total_num_scheduled_tokens = num_tokens
|
||||
mixed_output.num_common_prefix_blocks = [0] * num_kv_cache_groups
|
||||
|
||||
cleanup_output = SchedulerOutput.make_empty()
|
||||
cleanup_output.finished_req_ids = {decode_req_id, prefill_req_id}
|
||||
|
||||
context = mixed_step_context or nullcontext()
|
||||
model_runner.kv_connector.set_disabled(True)
|
||||
try:
|
||||
worker_execute_model(decode_prefill_output)
|
||||
worker_sample_tokens(None)
|
||||
with context:
|
||||
worker_execute_model(mixed_output)
|
||||
worker_sample_tokens(None)
|
||||
worker_execute_model(cleanup_output)
|
||||
finally:
|
||||
model_runner.kv_connector.set_disabled(False)
|
||||
return True
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def warmup_kernels(
|
||||
|
||||
Reference in New Issue
Block a user