[Model Runner V2] support qwen35 / mamba hybrid model (#35520)
Signed-off-by: zhuhaoran <zhuhaoran.zhr@alibaba-inc.com>
This commit is contained in:
@@ -44,10 +44,11 @@ from vllm.model_executor.model_loader import get_model_loader
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.tasks import SupportedTask
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.utils.mem_utils import DeviceMemoryProfiler, format_gib
|
||||
from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE
|
||||
from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig, MambaSpec
|
||||
from vllm.v1.outputs import DraftTokenIds, KVConnectorOutput, ModelRunnerOutput
|
||||
from vllm.v1.worker.cp_utils import check_attention_cp_compatibility
|
||||
from vllm.v1.worker.gpu.async_utils import AsyncOutput, AsyncPoolingOutput
|
||||
@@ -338,10 +339,6 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
def initialize_kv_cache(self, kv_cache_config: KVCacheConfig) -> None:
|
||||
kv_cache_config = deepcopy(kv_cache_config)
|
||||
self.kv_cache_config = kv_cache_config
|
||||
block_sizes = [
|
||||
kv_cache_group.kv_cache_spec.block_size
|
||||
for kv_cache_group in kv_cache_config.kv_cache_groups
|
||||
]
|
||||
|
||||
block_table_max_model_len = self.max_model_len
|
||||
if self.is_encoder_decoder:
|
||||
@@ -352,11 +349,34 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
getattr(self.model_config.hf_config, "max_source_positions", 0),
|
||||
)
|
||||
|
||||
block_sizes = []
|
||||
max_num_blocks_per_group = []
|
||||
for kv_cache_group in kv_cache_config.kv_cache_groups:
|
||||
spec = kv_cache_group.kv_cache_spec
|
||||
block_sizes.append(spec.block_size)
|
||||
# When using DCP, each request's KV cache is sharded among different ranks.
|
||||
# As a result, one block on the current rank covers `block_size * cp_size`
|
||||
# tokens in the full, global (unsharded) sequence.
|
||||
max_num_blocks = cdiv(
|
||||
block_table_max_model_len, spec.block_size * self.dcp_size
|
||||
)
|
||||
# Align to a multiple of (128 / block_size) as required by some attention
|
||||
# backends such as TRTLLM (#39324)
|
||||
if spec.block_size <= 128:
|
||||
alignment = 128 // spec.block_size
|
||||
max_num_blocks = cdiv(max_num_blocks, alignment) * alignment
|
||||
# For Mamba/Hybrid Model, KVCaches need extra blocks for speculative tokens
|
||||
if isinstance(spec, MambaSpec):
|
||||
max_num_blocks = (
|
||||
max_num_blocks if self.cache_config.enable_prefix_caching else 1
|
||||
) + spec.num_speculative_blocks
|
||||
max_num_blocks_per_group.append(max_num_blocks)
|
||||
|
||||
self.block_tables = BlockTables(
|
||||
block_sizes=block_sizes,
|
||||
max_num_reqs=self.max_num_reqs,
|
||||
max_num_batched_tokens=self.max_num_tokens,
|
||||
max_model_len=block_table_max_model_len,
|
||||
max_num_blocks_per_group=max_num_blocks_per_group,
|
||||
device=self.device,
|
||||
cp_size=self.dcp_size,
|
||||
cp_rank=self.dcp_rank,
|
||||
@@ -722,6 +742,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
|
||||
# Get the number of draft tokens for each request.
|
||||
draft_tokens = scheduler_output.scheduled_spec_decode_tokens
|
||||
num_draft_tokens_per_req = None
|
||||
if not draft_tokens:
|
||||
# No draft token scheduled (common case).
|
||||
total_num_draft_tokens = 0
|
||||
@@ -735,15 +756,15 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
num_reqs, dtype=torch.int32, device=self.device
|
||||
)
|
||||
else:
|
||||
num_draft_tokens = np.fromiter(
|
||||
num_draft_tokens_per_req = np.fromiter(
|
||||
(len(draft_tokens.get(req_id, ())) for req_id in req_ids),
|
||||
dtype=np.int32,
|
||||
count=num_reqs,
|
||||
)
|
||||
total_num_draft_tokens = int(num_draft_tokens.sum())
|
||||
total_num_draft_tokens = int(num_draft_tokens_per_req.sum())
|
||||
total_num_logits = num_reqs + total_num_draft_tokens
|
||||
|
||||
num_logits = num_draft_tokens + 1
|
||||
num_logits = num_draft_tokens_per_req + 1
|
||||
cu_num_logits_np = np.empty(num_reqs + 1, dtype=np.int32)
|
||||
cu_num_logits_np[0] = 0
|
||||
np.cumsum(num_logits, out=cu_num_logits_np[1:])
|
||||
@@ -766,9 +787,13 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
async_copy_to_gpu(query_start_loc_np, out=self.input_buffers.query_start_loc)
|
||||
query_start_loc_np = query_start_loc_np[: num_reqs_padded + 1]
|
||||
query_start_loc = self.input_buffers.query_start_loc[: num_reqs_padded + 1]
|
||||
is_prefilling_np = (
|
||||
self.req_states.num_computed_prefill_tokens[idx_mapping_np]
|
||||
< self.req_states.prefill_len.np[idx_mapping_np]
|
||||
)
|
||||
|
||||
# Get prefill tokens if any.
|
||||
if self.req_states.any_prefills(idx_mapping_np):
|
||||
if np.any(is_prefilling_np):
|
||||
prepare_prefill_inputs(
|
||||
self.input_buffers.input_ids,
|
||||
self.req_states.next_prefill_tokens,
|
||||
@@ -836,11 +861,13 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
num_tokens=num_tokens,
|
||||
num_tokens_after_padding=num_tokens_after_padding,
|
||||
num_draft_tokens=total_num_draft_tokens,
|
||||
num_draft_tokens_per_req=num_draft_tokens_per_req,
|
||||
query_start_loc=query_start_loc,
|
||||
query_start_loc_np=query_start_loc_np,
|
||||
seq_lens=seq_lens,
|
||||
seq_lens_cpu_upper_bound=seq_lens_cpu_upper_bound,
|
||||
dcp_local_seq_lens=dcp_local_seq_lens,
|
||||
is_prefilling_np=is_prefilling_np,
|
||||
input_ids=self.input_buffers.input_ids[:num_tokens_after_padding],
|
||||
positions=self.input_buffers.positions[:num_tokens_after_padding],
|
||||
logits_indices=logits_indices,
|
||||
@@ -958,6 +985,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
input_batch.num_scheduled_tokens
|
||||
)
|
||||
|
||||
self.model_state.postprocess_state(input_batch, num_sampled)
|
||||
|
||||
@torch.inference_mode()
|
||||
def execute_model(
|
||||
self,
|
||||
|
||||
Reference in New Issue
Block a user