[MRV2] Miscellaneous code cleanup (#57980)
This commit is contained in:
@@ -36,10 +36,7 @@ from vllm.distributed.aux_output_connector.worker import (
|
||||
AuxOutputWorkerConnector,
|
||||
get_aux_output_connector,
|
||||
)
|
||||
from vllm.distributed.parallel_state import (
|
||||
get_dcp_group,
|
||||
get_pp_group,
|
||||
)
|
||||
from vllm.distributed.parallel_state import get_dcp_group, get_pp_group
|
||||
from vllm.forward_context import BatchDescriptor, set_forward_context
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe.all2all_utils import get_ep_all2all_manager
|
||||
@@ -97,7 +94,7 @@ from vllm.v1.worker.gpu.attn_utils import (
|
||||
)
|
||||
from vllm.v1.worker.gpu.block_table import BlockTables
|
||||
from vllm.v1.worker.gpu.buffer_utils import set_default_max_concurrency
|
||||
from vllm.v1.worker.gpu.cp_utils import maybe_prepare_dcp_local_seq_lens
|
||||
from vllm.v1.worker.gpu.cp_utils import prepare_dcp_local_seq_lens
|
||||
from vllm.v1.worker.gpu.cudagraph_utils import (
|
||||
BatchExecutionDescriptor,
|
||||
ModelCudaGraphManager,
|
||||
@@ -494,8 +491,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
),
|
||||
)
|
||||
self.prompt_logprobs_worker = PromptLogprobsWorker(
|
||||
self.max_num_reqs,
|
||||
logprobs_mode=self.model_config.logprobs_mode,
|
||||
self.max_num_reqs, logprobs_mode=self.model_config.logprobs_mode
|
||||
)
|
||||
self.structured_outputs_worker = StructuredOutputsWorker(
|
||||
max_num_logits=self.max_num_reqs * self.decode_query_len,
|
||||
@@ -508,9 +504,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
if self.is_pooling_model and self.is_last_pp_rank:
|
||||
self.pooling_runner = PoolingRunner(self.model, self.vllm_config)
|
||||
eplb_models_added |= self.eplb.maybe_register_model(
|
||||
self.model,
|
||||
self.model_config,
|
||||
load_dummy_weights,
|
||||
self.model, self.model_config, load_dummy_weights
|
||||
)
|
||||
self.eplb.maybe_start_async_loop(eplb_models_added)
|
||||
|
||||
@@ -869,14 +863,10 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
self.step_timing.drafter_start()
|
||||
mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None
|
||||
if self.speculator.supports_mm_inputs:
|
||||
mm_inputs = (
|
||||
[],
|
||||
torch.zeros(
|
||||
input_batch.num_tokens,
|
||||
dtype=torch.bool,
|
||||
device="cpu",
|
||||
),
|
||||
all_false = torch.zeros(
|
||||
input_batch.num_tokens, dtype=torch.bool, device="cpu"
|
||||
)
|
||||
mm_inputs = [], all_false
|
||||
|
||||
# Let the target override the hidden state fed to the drafter
|
||||
# (e.g. DeepSeek V4 MTP needs the pre-hc_head residual). The
|
||||
@@ -945,9 +935,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
mm_config = self.model_config.multimodal_config
|
||||
if mm_config is not None and not mm_config.skip_mm_profiling:
|
||||
mm_budget = MultiModalBudget(
|
||||
self.vllm_config,
|
||||
self.mm_registry,
|
||||
enable_cache=False,
|
||||
self.vllm_config, self.mm_registry, enable_cache=False
|
||||
)
|
||||
dummy_mm_inputs = mm_budget.get_dummy_encoder_profile_inputs()
|
||||
self.model_state.encoder_runner.profile_encoder_cache(
|
||||
@@ -1294,7 +1282,9 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
req_ids = batch_req_state.req_ids
|
||||
num_scheduled_tokens_np = batch_req_state.num_scheduled_tokens
|
||||
idx_mapping_np = batch_req_state.idx_mapping_np
|
||||
idx_mapping = async_tensor_h2d(idx_mapping_np, device=self.device)
|
||||
idx_mapping = async_tensor_h2d(
|
||||
idx_mapping_np, device=self.device, dtype=torch.int32
|
||||
)
|
||||
num_reqs = len(req_ids)
|
||||
|
||||
# Get the number of draft tokens for each request.
|
||||
@@ -1471,11 +1461,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
else None
|
||||
),
|
||||
)
|
||||
input_batch = pcp.maybe_partition_pcp_batch(
|
||||
self.pcp_manager,
|
||||
input_batch,
|
||||
batch_desc,
|
||||
)
|
||||
if self.pcp_manager is not None:
|
||||
input_batch = self.pcp_manager.partition_batch(input_batch, batch_desc)
|
||||
return input_batch
|
||||
|
||||
def prepare_attn(
|
||||
@@ -1486,8 +1473,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
|
||||
# Block tables: num_kv_cache_groups x [num_reqs_padded, max_num_blocks].
|
||||
block_tables = self.block_tables.gather_block_tables(
|
||||
input_batch.idx_mapping,
|
||||
num_reqs_padded=input_batch.num_reqs_after_padding,
|
||||
input_batch.idx_mapping, num_reqs_padded=input_batch.num_reqs_after_padding
|
||||
)
|
||||
# Slot mappings: [num_kv_cache_groups, num_tokens_padded].
|
||||
# Kernel pads beyond num_tokens with PAD_SLOT_ID.
|
||||
@@ -1506,15 +1492,12 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
block_tables = block_table_provider.get_dummy_block_tables(input_batch.num_reqs)
|
||||
if valid_state_slots:
|
||||
state_slots = torch.arange(
|
||||
1,
|
||||
input_batch.num_reqs + 1,
|
||||
dtype=torch.int32,
|
||||
device=self.device,
|
||||
1, input_batch.num_reqs + 1, dtype=torch.int32, device=self.device
|
||||
)
|
||||
for block_table in block_tables:
|
||||
block_table[:, 0].copy_(state_slots)
|
||||
slot_mappings = pcp.maybe_get_pcp_dummy_slot_mappings(
|
||||
self.pcp_manager, self.block_tables, input_batch.num_tokens
|
||||
slot_mappings = block_table_provider.get_dummy_slot_mappings(
|
||||
input_batch.num_tokens
|
||||
)
|
||||
return block_tables, slot_mappings
|
||||
|
||||
@@ -1607,11 +1590,10 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
query_start_loc: torch.Tensor | None = None,
|
||||
) -> None:
|
||||
# Update the number of computed tokens.
|
||||
output_bin_counts = None
|
||||
if self.is_last_pp_rank:
|
||||
assert self.sampler is not None
|
||||
output_bin_counts = self.sampler.penalties_state.output_bin_counts
|
||||
else:
|
||||
output_bin_counts = None
|
||||
post_update(
|
||||
idx_mapping,
|
||||
self.req_states.num_computed_tokens.gpu,
|
||||
@@ -1769,32 +1751,27 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
"Attention metadata must be prepared for dummy runs when using "
|
||||
"FULL cudagraph mode."
|
||||
)
|
||||
block_tables = None
|
||||
slot_mappings = None
|
||||
block_tables, slot_mappings = None, None
|
||||
else:
|
||||
block_tables, slot_mappings = self.prepare_dummy_attn(
|
||||
input_batch, valid_dummy_state_slots
|
||||
)
|
||||
if not skip_attn_for_dummy_run and context_len:
|
||||
assert block_tables is not None
|
||||
set_dummy_context(
|
||||
input_batch,
|
||||
self.block_tables,
|
||||
context_len,
|
||||
self.kv_cache_config.num_blocks,
|
||||
self.max_model_len,
|
||||
input_block_tables=block_tables,
|
||||
)
|
||||
if context_len:
|
||||
set_dummy_context(
|
||||
input_batch,
|
||||
self.block_tables,
|
||||
context_len,
|
||||
self.kv_cache_config.num_blocks,
|
||||
self.max_model_len,
|
||||
input_block_tables=block_tables,
|
||||
)
|
||||
|
||||
attn_metadata = None
|
||||
slot_mappings_by_layer = None
|
||||
if not (dummy_run and skip_attn_for_dummy_run):
|
||||
input_batch.dcp_local_seq_lens = maybe_prepare_dcp_local_seq_lens(
|
||||
(
|
||||
self.pcp_manager.input_buffers
|
||||
if self.pcp_manager is not None
|
||||
else self.input_buffers
|
||||
).dcp_local_seq_lens,
|
||||
if self.dcp_size > 1 and not (dummy_run and skip_attn_for_dummy_run):
|
||||
input_buffers = self.input_buffers
|
||||
if self.pcp_manager is not None:
|
||||
input_buffers = self.pcp_manager.input_buffers
|
||||
input_batch.dcp_local_seq_lens = prepare_dcp_local_seq_lens(
|
||||
input_buffers.dcp_local_seq_lens,
|
||||
input_batch.seq_lens,
|
||||
input_batch.num_reqs,
|
||||
self.dcp_size,
|
||||
@@ -1802,6 +1779,9 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
self.cp_interleave,
|
||||
num_reqs_padded=input_batch.num_reqs_after_padding,
|
||||
)
|
||||
|
||||
attn_metadata = None
|
||||
slot_mappings_by_layer = None
|
||||
ubatch_state: UBatchState | None = None
|
||||
if batch_desc.num_ubatches > 1:
|
||||
assert self.ubatch_runner is not None
|
||||
@@ -2046,16 +2026,12 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
return ModelRunnerOutput.with_ec_conn_output(output, ec_connector_output)
|
||||
|
||||
# Last rank: sample tokens
|
||||
assert hidden_states is not None
|
||||
draft_hidden_states = hidden_states
|
||||
assert draft_hidden_states is not None
|
||||
hidden_states, input_batch = pcp.maybe_restore_pcp_for_sampling(
|
||||
self.pcp_manager, hidden_states, input_batch
|
||||
)
|
||||
if self.pcp_manager is not None and aux_hidden_states is not None:
|
||||
aux_hidden_states = [
|
||||
self.pcp_manager.restore_hidden_states(states)
|
||||
for states in aux_hidden_states
|
||||
]
|
||||
if self.pcp_manager is not None:
|
||||
hidden_states, aux_hidden_states, input_batch = (
|
||||
self.pcp_manager.restore_for_sampling(hidden_states, aux_hidden_states)
|
||||
)
|
||||
|
||||
sampler_output, num_sampled, num_rejected = self.sample(
|
||||
hidden_states, input_batch, grammar_output
|
||||
@@ -2305,13 +2281,10 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
self.eplb.suppressed = suppressed
|
||||
|
||||
def setup_eplb_from_mapping(
|
||||
self,
|
||||
expanded_physical_to_logical: torch.Tensor,
|
||||
self, expanded_physical_to_logical: torch.Tensor
|
||||
) -> None:
|
||||
self.eplb.setup_from_mapping(
|
||||
self.model,
|
||||
self.model_config,
|
||||
expanded_physical_to_logical,
|
||||
self.model, self.model_config, expanded_physical_to_logical
|
||||
)
|
||||
|
||||
########### EPLB methods end ###########
|
||||
|
||||
Reference in New Issue
Block a user