[Model Runner V2] support qwen35 / mamba hybrid model (#35520)

Signed-off-by: zhuhaoran <zhuhaoran.zhr@alibaba-inc.com>
This commit is contained in:
zhrrr
2026-05-07 09:31:05 -07:00
committed by GitHub
parent 06a60d3dd0
commit 7a08b34fbf
14 changed files with 439 additions and 106 deletions
+39 -10
View File
@@ -44,10 +44,11 @@ from vllm.model_executor.model_loader import get_model_loader
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.sequence import IntermediateTensors
from vllm.tasks import SupportedTask
from vllm.utils.math_utils import cdiv
from vllm.utils.mem_utils import DeviceMemoryProfiler, format_gib
from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE
from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.kv_cache_interface import KVCacheConfig, MambaSpec
from vllm.v1.outputs import DraftTokenIds, KVConnectorOutput, ModelRunnerOutput
from vllm.v1.worker.cp_utils import check_attention_cp_compatibility
from vllm.v1.worker.gpu.async_utils import AsyncOutput, AsyncPoolingOutput
@@ -338,10 +339,6 @@ class GPUModelRunner(LoRAModelRunnerMixin):
def initialize_kv_cache(self, kv_cache_config: KVCacheConfig) -> None:
kv_cache_config = deepcopy(kv_cache_config)
self.kv_cache_config = kv_cache_config
block_sizes = [
kv_cache_group.kv_cache_spec.block_size
for kv_cache_group in kv_cache_config.kv_cache_groups
]
block_table_max_model_len = self.max_model_len
if self.is_encoder_decoder:
@@ -352,11 +349,34 @@ class GPUModelRunner(LoRAModelRunnerMixin):
getattr(self.model_config.hf_config, "max_source_positions", 0),
)
block_sizes = []
max_num_blocks_per_group = []
for kv_cache_group in kv_cache_config.kv_cache_groups:
spec = kv_cache_group.kv_cache_spec
block_sizes.append(spec.block_size)
# When using DCP, each request's KV cache is sharded among different ranks.
# As a result, one block on the current rank covers `block_size * cp_size`
# tokens in the full, global (unsharded) sequence.
max_num_blocks = cdiv(
block_table_max_model_len, spec.block_size * self.dcp_size
)
# Align to a multiple of (128 / block_size) as required by some attention
# backends such as TRTLLM (#39324)
if spec.block_size <= 128:
alignment = 128 // spec.block_size
max_num_blocks = cdiv(max_num_blocks, alignment) * alignment
# For Mamba/Hybrid Model, KVCaches need extra blocks for speculative tokens
if isinstance(spec, MambaSpec):
max_num_blocks = (
max_num_blocks if self.cache_config.enable_prefix_caching else 1
) + spec.num_speculative_blocks
max_num_blocks_per_group.append(max_num_blocks)
self.block_tables = BlockTables(
block_sizes=block_sizes,
max_num_reqs=self.max_num_reqs,
max_num_batched_tokens=self.max_num_tokens,
max_model_len=block_table_max_model_len,
max_num_blocks_per_group=max_num_blocks_per_group,
device=self.device,
cp_size=self.dcp_size,
cp_rank=self.dcp_rank,
@@ -722,6 +742,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
# Get the number of draft tokens for each request.
draft_tokens = scheduler_output.scheduled_spec_decode_tokens
num_draft_tokens_per_req = None
if not draft_tokens:
# No draft token scheduled (common case).
total_num_draft_tokens = 0
@@ -735,15 +756,15 @@ class GPUModelRunner(LoRAModelRunnerMixin):
num_reqs, dtype=torch.int32, device=self.device
)
else:
num_draft_tokens = np.fromiter(
num_draft_tokens_per_req = np.fromiter(
(len(draft_tokens.get(req_id, ())) for req_id in req_ids),
dtype=np.int32,
count=num_reqs,
)
total_num_draft_tokens = int(num_draft_tokens.sum())
total_num_draft_tokens = int(num_draft_tokens_per_req.sum())
total_num_logits = num_reqs + total_num_draft_tokens
num_logits = num_draft_tokens + 1
num_logits = num_draft_tokens_per_req + 1
cu_num_logits_np = np.empty(num_reqs + 1, dtype=np.int32)
cu_num_logits_np[0] = 0
np.cumsum(num_logits, out=cu_num_logits_np[1:])
@@ -766,9 +787,13 @@ class GPUModelRunner(LoRAModelRunnerMixin):
async_copy_to_gpu(query_start_loc_np, out=self.input_buffers.query_start_loc)
query_start_loc_np = query_start_loc_np[: num_reqs_padded + 1]
query_start_loc = self.input_buffers.query_start_loc[: num_reqs_padded + 1]
is_prefilling_np = (
self.req_states.num_computed_prefill_tokens[idx_mapping_np]
< self.req_states.prefill_len.np[idx_mapping_np]
)
# Get prefill tokens if any.
if self.req_states.any_prefills(idx_mapping_np):
if np.any(is_prefilling_np):
prepare_prefill_inputs(
self.input_buffers.input_ids,
self.req_states.next_prefill_tokens,
@@ -836,11 +861,13 @@ class GPUModelRunner(LoRAModelRunnerMixin):
num_tokens=num_tokens,
num_tokens_after_padding=num_tokens_after_padding,
num_draft_tokens=total_num_draft_tokens,
num_draft_tokens_per_req=num_draft_tokens_per_req,
query_start_loc=query_start_loc,
query_start_loc_np=query_start_loc_np,
seq_lens=seq_lens,
seq_lens_cpu_upper_bound=seq_lens_cpu_upper_bound,
dcp_local_seq_lens=dcp_local_seq_lens,
is_prefilling_np=is_prefilling_np,
input_ids=self.input_buffers.input_ids[:num_tokens_after_padding],
positions=self.input_buffers.positions[:num_tokens_after_padding],
logits_indices=logits_indices,
@@ -958,6 +985,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
input_batch.num_scheduled_tokens
)
self.model_state.postprocess_state(input_batch, num_sampled)
@torch.inference_mode()
def execute_model(
self,