[MRV2] Miscellaneous code cleanup (#57980)
This commit is contained in:
@@ -255,9 +255,7 @@ class AdaptiveVerificationManager:
|
||||
logger.debug("DSpark cost tables: %s", self.cost_tables)
|
||||
|
||||
def record_confidences(
|
||||
self,
|
||||
confidence_probs: torch.Tensor,
|
||||
input_batch: "InputBatch",
|
||||
self, confidence_probs: torch.Tensor, input_batch: "InputBatch"
|
||||
) -> None:
|
||||
"""Publish this step's raw confidences for the ranking kernel and start
|
||||
copying them to the CPU, where a later step's budget reads them."""
|
||||
@@ -305,9 +303,9 @@ class AdaptiveVerificationManager:
|
||||
)
|
||||
num_non_draft_tokens = scheduled_tokens - scheduled_drafts
|
||||
slots = np.fromiter(
|
||||
(self.req_states.req_id_to_index[req_id] for req_id in req_ids),
|
||||
map(self.req_states.req_id_to_index.__getitem__, req_ids),
|
||||
dtype=np.int32,
|
||||
count=len(req_ids),
|
||||
count=num_reqs,
|
||||
)
|
||||
stale_confidences = self._stale_confidences[self._stale_idx].np[slots]
|
||||
survival_probability = np.cumprod(stale_confidences.astype(np.float64), axis=1)
|
||||
@@ -469,8 +467,7 @@ def maybe_create_adaptive_verification_manager(
|
||||
# The selector rejects unsupported backends, but models that
|
||||
# hard-wire theirs (e.g. DeepSeek-V4) never go through it.
|
||||
backend = get_query_lens_mismatch_unsupported_backend(
|
||||
attn_groups,
|
||||
checked_layer_names=target_layer_names,
|
||||
attn_groups, checked_layer_names=target_layer_names
|
||||
)
|
||||
if backend is not None:
|
||||
raise ValueError(
|
||||
|
||||
@@ -53,10 +53,7 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
|
||||
return
|
||||
|
||||
self.inputs_embeds = torch.zeros(
|
||||
self.max_num_tokens,
|
||||
self.hidden_size,
|
||||
dtype=self.dtype,
|
||||
device=self.device,
|
||||
self.max_num_tokens, self.hidden_size, dtype=self.dtype, device=self.device
|
||||
)
|
||||
|
||||
# Lifecycle hooks for model-specific optimizations. Subclasses override
|
||||
@@ -140,10 +137,7 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
|
||||
|
||||
# Initialize cudagraph manager for draft decodes (draft positions > 0).
|
||||
self.decode_cudagraph_manager = SpeculatorCudaGraphManager(
|
||||
self.vllm_config,
|
||||
self.device,
|
||||
cudagraph_mode,
|
||||
decode_query_len=1,
|
||||
self.vllm_config, self.device, cudagraph_mode, decode_query_len=1
|
||||
)
|
||||
|
||||
def capture(self) -> None:
|
||||
@@ -272,8 +266,7 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
|
||||
|
||||
if self.pcp_manager is not None:
|
||||
self.pcp_manager.prepare_draft_prefill(
|
||||
input_batch,
|
||||
self.input_buffers.input_ids[:num_tokens_padded],
|
||||
input_batch, self.input_buffers.input_ids[:num_tokens_padded]
|
||||
)
|
||||
prefill = self.pcp_manager.draft_prefill_batch
|
||||
if prefill is not None:
|
||||
|
||||
@@ -11,7 +11,7 @@ from vllm.v1.worker.gpu.attn_utils import (
|
||||
build_slot_mappings_by_layer,
|
||||
)
|
||||
from vllm.v1.worker.gpu.block_table import BlockTables
|
||||
from vllm.v1.worker.gpu.cp_utils import maybe_prepare_dcp_local_seq_lens
|
||||
from vllm.v1.worker.gpu.cp_utils import prepare_dcp_local_seq_lens
|
||||
from vllm.v1.worker.gpu.cudagraph_utils import (
|
||||
AttentionState,
|
||||
BatchExecutionDescriptor,
|
||||
@@ -42,15 +42,16 @@ def _prepare_dflash_inputs_to_capture(
|
||||
attn_metadata = None
|
||||
if not skip_attn:
|
||||
query_start_loc_cpu = torch.from_numpy(input_batch.query_start_loc_np)
|
||||
input_batch.dcp_local_seq_lens = maybe_prepare_dcp_local_seq_lens(
|
||||
input_buffers.dcp_local_seq_lens,
|
||||
input_batch.seq_lens,
|
||||
input_batch.num_reqs,
|
||||
block_tables.cp_size,
|
||||
block_tables.cp_rank,
|
||||
block_tables.cp_interleave,
|
||||
num_reqs_padded=input_batch.num_reqs_after_padding,
|
||||
)
|
||||
if block_tables.cp_size > 1:
|
||||
input_batch.dcp_local_seq_lens = prepare_dcp_local_seq_lens(
|
||||
input_buffers.dcp_local_seq_lens,
|
||||
input_batch.seq_lens,
|
||||
input_batch.num_reqs,
|
||||
block_tables.cp_size,
|
||||
block_tables.cp_rank,
|
||||
block_tables.cp_interleave,
|
||||
num_reqs_padded=input_batch.num_reqs_after_padding,
|
||||
)
|
||||
attn_metadata = build_attn_metadata(
|
||||
attn_groups=attn_groups,
|
||||
num_reqs=num_reqs,
|
||||
|
||||
@@ -149,10 +149,7 @@ class DSparkSpeculator(DFlashSpeculator):
|
||||
)
|
||||
if self.draft_watermarker is not None:
|
||||
sampled = self.draft_watermarker.sample(
|
||||
logits,
|
||||
sampled,
|
||||
idx_map,
|
||||
self.temperature,
|
||||
logits, sampled, idx_map, self.temperature
|
||||
)
|
||||
return sampled
|
||||
|
||||
|
||||
@@ -134,8 +134,7 @@ class RejectionSampler:
|
||||
if expanded_logits:
|
||||
if self.enable_adaptive_verification:
|
||||
# Adaptive verification keeps the true per-request boundaries
|
||||
# on device only; cu_num_logits_np holds the pre-compacted
|
||||
# layout.
|
||||
# on device only; cu_num_logits_np holds the pre-compacted layout.
|
||||
cu_num_generated_tokens = cu_num_logits.clone()
|
||||
else:
|
||||
cu_num_generated_tokens = cu_num_logits_np.tolist()
|
||||
|
||||
@@ -26,7 +26,7 @@ from vllm.v1.worker.gpu.attn_utils import (
|
||||
init_attn_backend,
|
||||
)
|
||||
from vllm.v1.worker.gpu.block_table import BlockTables
|
||||
from vllm.v1.worker.gpu.cp_utils import maybe_prepare_dcp_local_seq_lens
|
||||
from vllm.v1.worker.gpu.cp_utils import prepare_dcp_local_seq_lens
|
||||
from vllm.v1.worker.gpu.cudagraph_utils import BatchExecutionDescriptor
|
||||
from vllm.v1.worker.gpu.dp_utils import DPSyncState
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers
|
||||
@@ -202,9 +202,7 @@ class DraftModelSpeculator(BaseSpeculator):
|
||||
|
||||
@abstractmethod
|
||||
def load_draft_model(
|
||||
self,
|
||||
target_model: nn.Module,
|
||||
target_attn_layer_names: set[str],
|
||||
self, target_model: nn.Module, target_attn_layer_names: set[str]
|
||||
) -> nn.Module:
|
||||
pass
|
||||
|
||||
@@ -337,7 +335,7 @@ class DraftModelSpeculator(BaseSpeculator):
|
||||
if dcp_local_seq_lens is None and self.block_tables.cp_size > 1:
|
||||
# Draft steps advance and rewind their own global sequence lengths,
|
||||
# so the target model's DCP-local lengths may already be stale.
|
||||
dcp_local_seq_lens = maybe_prepare_dcp_local_seq_lens(
|
||||
dcp_local_seq_lens = prepare_dcp_local_seq_lens(
|
||||
self.input_buffers.dcp_local_seq_lens,
|
||||
self.input_buffers.seq_lens,
|
||||
num_reqs,
|
||||
@@ -422,10 +420,7 @@ class DraftModelSpeculator(BaseSpeculator):
|
||||
)
|
||||
if self.draft_watermarker is not None:
|
||||
sampled = self.draft_watermarker.sample(
|
||||
logits,
|
||||
sampled,
|
||||
idx_mapping,
|
||||
temperature,
|
||||
logits, sampled, idx_mapping, temperature
|
||||
)
|
||||
elif self.use_local_argmax_reduction:
|
||||
return self.model.get_top_tokens(hidden_states)
|
||||
@@ -436,10 +431,7 @@ class DraftModelSpeculator(BaseSpeculator):
|
||||
return sampled
|
||||
|
||||
def _maybe_predict_acceptance(
|
||||
self,
|
||||
logits: torch.Tensor,
|
||||
idx_mapping: torch.Tensor,
|
||||
draft_step: torch.Tensor,
|
||||
self, logits: torch.Tensor, idx_mapping: torch.Tensor, draft_step: torch.Tensor
|
||||
) -> None:
|
||||
if self.acceptance_estimator is not None:
|
||||
self.acceptance_estimator.predict(
|
||||
|
||||
Reference in New Issue
Block a user