[MRV2] Revert explicit Triton JIT warmup migration (#56654)
Signed-off-by: Woosuk Kwon <woosuk@inferact.ai> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
c711f740b7
commit
2671fedfc7
@@ -188,7 +188,6 @@ logger = init_logger(__name__)
|
||||
|
||||
|
||||
class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
@JitWarmupRegistry.capture
|
||||
def __init__(self, vllm_config: VllmConfig, device: torch.device):
|
||||
self.vllm_config = vllm_config
|
||||
self.model_config = vllm_config.model_config
|
||||
@@ -203,6 +202,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
self.speculative_config is not None and self.speculative_config.use_dspark()
|
||||
)
|
||||
self.observability_config = vllm_config.observability_config
|
||||
self.jit_warmup_registry = JitWarmupRegistry(vllm_config)
|
||||
|
||||
self.device = device
|
||||
self.dtype = self.model_config.dtype
|
||||
@@ -399,7 +399,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
logger.info_once("Loading model from scratch...")
|
||||
|
||||
# Capture warmup providers selected while constructing the model.
|
||||
with self.jit_warmup_registry.activate(): # type: ignore[attr-defined]
|
||||
with self.jit_warmup_registry.activate():
|
||||
self.model = model_loader.load_model(
|
||||
vllm_config=self.vllm_config,
|
||||
model_config=self.vllm_config.model_config,
|
||||
@@ -461,50 +461,47 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
|
||||
# Initialize samplers. Model states may override via custom_sampler().
|
||||
if self.is_last_pp_rank and not self.is_pooling_model:
|
||||
with self.jit_warmup_registry.activate(): # type: ignore[attr-defined]
|
||||
sampler_kwargs: dict[str, Any] = {
|
||||
"max_num_reqs": self.max_num_reqs,
|
||||
"vocab_size": self.vocab_size,
|
||||
"device": self.device,
|
||||
"req_states": self.req_states,
|
||||
"logprobs_mode": self.model_config.logprobs_mode,
|
||||
"num_speculative_tokens": self.decode_query_len,
|
||||
"use_fp64_gumbel": self.model_config.use_fp64_gumbel,
|
||||
"enable_trace_replay": self.model_config.enable_trace_replay,
|
||||
"reasoning_config": self.vllm_config.reasoning_config,
|
||||
"return_sampling_mask": self.model_config.return_sampling_mask,
|
||||
}
|
||||
if self.vllm_config.watermark_config is None:
|
||||
self.sampler = Sampler(**sampler_kwargs)
|
||||
else:
|
||||
wm_config = self.vllm_config.watermark_config
|
||||
watermarker = create_watermarker(wm_config)
|
||||
if self.speculative_config is not None:
|
||||
watermarker = create_speculative_target_watermarker(watermarker)
|
||||
self.sampler = GPUWatermarkSampler(
|
||||
watermarker,
|
||||
deduplicate_contexts=wm_config.deduplicate_contexts,
|
||||
deduplicate_contexts_max_history=(
|
||||
wm_config.deduplicate_contexts_max_history
|
||||
),
|
||||
**sampler_kwargs,
|
||||
)
|
||||
custom = self.model_state.custom_sampler(self.sampler)
|
||||
sampler_kwargs: dict[str, Any] = {
|
||||
"max_num_reqs": self.max_num_reqs,
|
||||
"vocab_size": self.vocab_size,
|
||||
"device": self.device,
|
||||
"req_states": self.req_states,
|
||||
"logprobs_mode": self.model_config.logprobs_mode,
|
||||
"num_speculative_tokens": self.decode_query_len,
|
||||
"use_fp64_gumbel": self.model_config.use_fp64_gumbel,
|
||||
"enable_trace_replay": self.model_config.enable_trace_replay,
|
||||
"reasoning_config": self.vllm_config.reasoning_config,
|
||||
"return_sampling_mask": self.model_config.return_sampling_mask,
|
||||
}
|
||||
if self.vllm_config.watermark_config is None:
|
||||
self.sampler = Sampler(**sampler_kwargs)
|
||||
else:
|
||||
wm_config = self.vllm_config.watermark_config
|
||||
watermarker = create_watermarker(wm_config)
|
||||
if self.speculative_config is not None:
|
||||
watermarker = create_speculative_target_watermarker(watermarker)
|
||||
self.sampler = GPUWatermarkSampler(
|
||||
watermarker,
|
||||
deduplicate_contexts=wm_config.deduplicate_contexts,
|
||||
deduplicate_contexts_max_history=(
|
||||
wm_config.deduplicate_contexts_max_history
|
||||
),
|
||||
**sampler_kwargs,
|
||||
)
|
||||
custom = self.model_state.custom_sampler(self.sampler)
|
||||
|
||||
if custom:
|
||||
self.vllm_config._check_watermarking_unsupported(
|
||||
custom_sampler=True
|
||||
)
|
||||
self.sampler, self.rejection_sampler = custom
|
||||
elif self.speculative_config is not None:
|
||||
self.rejection_sampler = RejectionSampler(
|
||||
self.sampler,
|
||||
self.speculative_config,
|
||||
self.device,
|
||||
watermark_key=speculative_target_watermark_key(
|
||||
self.vllm_config.watermark_config
|
||||
),
|
||||
)
|
||||
if custom:
|
||||
self.vllm_config._check_watermarking_unsupported(custom_sampler=True)
|
||||
self.sampler, self.rejection_sampler = custom
|
||||
elif self.speculative_config is not None:
|
||||
self.rejection_sampler = RejectionSampler(
|
||||
self.sampler,
|
||||
self.speculative_config,
|
||||
self.device,
|
||||
watermark_key=speculative_target_watermark_key(
|
||||
self.vllm_config.watermark_config
|
||||
),
|
||||
)
|
||||
self.prompt_logprobs_worker = PromptLogprobsWorker(
|
||||
self.max_num_reqs,
|
||||
logprobs_mode=self.model_config.logprobs_mode,
|
||||
@@ -741,7 +738,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
self.speculator.init_cudagraph_manager(cudagraph_mode)
|
||||
|
||||
# Capture warmup providers that depend on allocated KV-cache strides.
|
||||
with self.jit_warmup_registry.activate(): # type: ignore[attr-defined]
|
||||
with self.jit_warmup_registry.activate():
|
||||
kv_caches_dict = init_kv_cache(
|
||||
self.compilation_config.static_forward_context,
|
||||
self.kv_cache_config,
|
||||
@@ -927,12 +924,6 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
# during actual execution.
|
||||
assert self.sampler is not None
|
||||
self.sampler(logits, dummy_input_batch)
|
||||
if self.rejection_sampler is not None:
|
||||
# Warm head-dtype logits; scheduler warmup covers processed FP32 logits.
|
||||
assert self.speculator is not None
|
||||
self.rejection_sampler(
|
||||
logits, dummy_input_batch, self.speculator.draft_logits
|
||||
)
|
||||
|
||||
@torch.inference_mode()
|
||||
def _dummy_pooler_run(self, hidden_states: torch.Tensor) -> None:
|
||||
|
||||
Reference in New Issue
Block a user