[ModelRunnerV2] Various model/config compatibility fixes (#45868)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -2026,6 +2026,14 @@ class VllmConfig:
|
|||||||
):
|
):
|
||||||
unsupported.append("sequence parallelism")
|
unsupported.append("sequence parallelism")
|
||||||
|
|
||||||
|
# V2 does not implement the external_launcher (torchrun) PP-output
|
||||||
|
# broadcast that V1 uses to keep all ranks in sync (broadcast_pp_output).
|
||||||
|
if (
|
||||||
|
self.parallel_config.distributed_executor_backend == "external_launcher"
|
||||||
|
and self.parallel_config.pipeline_parallel_size > 1
|
||||||
|
):
|
||||||
|
unsupported.append("pipeline parallelism with external_launcher")
|
||||||
|
|
||||||
if speculative_config is not None:
|
if speculative_config is not None:
|
||||||
# TODO: ngram / ngram_gpu are not supported by the v2 model runner yet
|
# TODO: ngram / ngram_gpu are not supported by the v2 model runner yet
|
||||||
if speculative_config.method in ("ngram", "ngram_gpu"):
|
if speculative_config.method in ("ngram", "ngram_gpu"):
|
||||||
|
|||||||
@@ -124,7 +124,7 @@ class EncoderRunner:
|
|||||||
mm_embeds_item = encoder_output[start_idx:end_idx]
|
mm_embeds_item = encoder_output[start_idx:end_idx]
|
||||||
|
|
||||||
req_start_pos = query_start_loc[i] + start_pos - query_start[i]
|
req_start_pos = query_start_loc[i] + start_pos - query_start[i]
|
||||||
is_mm_embed[req_start_pos + start_idx : req_start_pos + end_idx] = (
|
is_mm_embed[req_start_pos + start_idx : req_start_pos + end_idx] |= (
|
||||||
True if is_embed is None else is_embed
|
True if is_embed is None else is_embed
|
||||||
)
|
)
|
||||||
mm_embeds.append(mm_embeds_item)
|
mm_embeds.append(mm_embeds_item)
|
||||||
|
|||||||
@@ -1204,6 +1204,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
|||||||
self.kv_cache_config,
|
self.kv_cache_config,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
input_ids = input_batch.input_ids
|
||||||
inputs_embeds = None
|
inputs_embeds = None
|
||||||
if self.supports_mm_inputs and self.is_first_pp_rank:
|
if self.supports_mm_inputs and self.is_first_pp_rank:
|
||||||
# Run MM encoder (if needed) and get multimodal embeddings.
|
# Run MM encoder (if needed) and get multimodal embeddings.
|
||||||
@@ -1222,11 +1223,14 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
|||||||
inputs_embeds = self.model_state.get_mm_embeddings(
|
inputs_embeds = self.model_state.get_mm_embeddings(
|
||||||
scheduler_output.scheduled_encoder_inputs, input_batch
|
scheduler_output.scheduled_encoder_inputs, input_batch
|
||||||
)
|
)
|
||||||
|
if inputs_embeds is not None and not self.model.requires_raw_input_tokens:
|
||||||
|
input_ids = None
|
||||||
|
|
||||||
model_inputs = {
|
model_inputs = {
|
||||||
"input_ids": input_batch.input_ids,
|
"input_ids": input_ids,
|
||||||
"positions": input_batch.positions,
|
"positions": input_batch.positions,
|
||||||
"inputs_embeds": inputs_embeds,
|
"inputs_embeds": inputs_embeds,
|
||||||
|
"intermediate_tensors": None,
|
||||||
# NOTE: Values returned by `prepare_inputs` will override the default
|
# NOTE: Values returned by `prepare_inputs` will override the default
|
||||||
# values above.
|
# values above.
|
||||||
**self.model_state.prepare_inputs(input_batch, self.req_states),
|
**self.model_state.prepare_inputs(input_batch, self.req_states),
|
||||||
|
|||||||
@@ -132,7 +132,9 @@ class WhisperModelState(ModelState):
|
|||||||
num_reqs = input_batch.num_reqs
|
num_reqs = input_batch.num_reqs
|
||||||
num_tokens = input_batch.num_tokens
|
num_tokens = input_batch.num_tokens
|
||||||
whisper_attn_metadata = WhisperAttnMetadata(
|
whisper_attn_metadata = WhisperAttnMetadata(
|
||||||
self._get_encoder_seq_lens(input_batch.req_ids, attn_groups, for_capture)
|
self._get_encoder_seq_lens(
|
||||||
|
input_batch.req_ids, attn_groups, for_capture, num_reqs
|
||||||
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
query_start_loc_cpu = torch.from_numpy(input_batch.query_start_loc_np)
|
query_start_loc_cpu = torch.from_numpy(input_batch.query_start_loc_np)
|
||||||
@@ -166,8 +168,8 @@ class WhisperModelState(ModelState):
|
|||||||
req_ids: list[str],
|
req_ids: list[str],
|
||||||
attn_groups: list[list[AttentionGroup]],
|
attn_groups: list[list[AttentionGroup]],
|
||||||
for_capture: bool,
|
for_capture: bool,
|
||||||
|
num_reqs: int,
|
||||||
) -> dict[int, tuple[torch.Tensor, np.ndarray]]:
|
) -> dict[int, tuple[torch.Tensor, np.ndarray]]:
|
||||||
num_reqs = len(req_ids)
|
|
||||||
encoder_seq_lens_np = np.zeros(num_reqs, dtype=np.int32)
|
encoder_seq_lens_np = np.zeros(num_reqs, dtype=np.int32)
|
||||||
if not for_capture:
|
if not for_capture:
|
||||||
# During normal execution, use actual encoder lengths.
|
# During normal execution, use actual encoder lengths.
|
||||||
|
|||||||
Reference in New Issue
Block a user