[6/N][warmup][DSv4] Migrate sampling, and DFlash JIT kernels (#56323)

Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Signed-off-by: LopezCastroRoberto <rocastro@redhat.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Roberto L. Castro
2026-09-12 17:24:24 -07:00
committed by GitHub
co-authored by Lucas Wilkinson OpenAI Codex
parent e19a3e172e
commit aed894c190
27 changed files with 2288 additions and 558 deletions
+52 -43
View File
@@ -188,6 +188,7 @@ logger = init_logger(__name__)
class GPUModelRunner(LoRAModelRunnerMixin):
@JitWarmupRegistry.capture
def __init__(self, vllm_config: VllmConfig, device: torch.device):
self.vllm_config = vllm_config
self.model_config = vllm_config.model_config
@@ -202,7 +203,6 @@ class GPUModelRunner(LoRAModelRunnerMixin):
self.speculative_config is not None and self.speculative_config.use_dspark()
)
self.observability_config = vllm_config.observability_config
self.jit_warmup_registry = JitWarmupRegistry(vllm_config)
self.device = device
self.dtype = self.model_config.dtype
@@ -399,7 +399,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
logger.info_once("Loading model from scratch...")
# Capture warmup providers selected while constructing the model.
with self.jit_warmup_registry.activate():
with self.jit_warmup_registry.activate(): # type: ignore[attr-defined]
self.model = model_loader.load_model(
vllm_config=self.vllm_config,
model_config=self.vllm_config.model_config,
@@ -461,47 +461,50 @@ class GPUModelRunner(LoRAModelRunnerMixin):
# Initialize samplers. Model states may override via custom_sampler().
if self.is_last_pp_rank and not self.is_pooling_model:
sampler_kwargs: dict[str, Any] = {
"max_num_reqs": self.max_num_reqs,
"vocab_size": self.vocab_size,
"device": self.device,
"req_states": self.req_states,
"logprobs_mode": self.model_config.logprobs_mode,
"num_speculative_tokens": self.decode_query_len,
"use_fp64_gumbel": self.model_config.use_fp64_gumbel,
"enable_trace_replay": self.model_config.enable_trace_replay,
"reasoning_config": self.vllm_config.reasoning_config,
"return_sampling_mask": self.model_config.return_sampling_mask,
}
if self.vllm_config.watermark_config is None:
self.sampler = Sampler(**sampler_kwargs)
else:
wm_config = self.vllm_config.watermark_config
watermarker = create_watermarker(wm_config)
if self.speculative_config is not None:
watermarker = create_speculative_target_watermarker(watermarker)
self.sampler = GPUWatermarkSampler(
watermarker,
deduplicate_contexts=wm_config.deduplicate_contexts,
deduplicate_contexts_max_history=(
wm_config.deduplicate_contexts_max_history
),
**sampler_kwargs,
)
custom = self.model_state.custom_sampler(self.sampler)
with self.jit_warmup_registry.activate(): # type: ignore[attr-defined]
sampler_kwargs: dict[str, Any] = {
"max_num_reqs": self.max_num_reqs,
"vocab_size": self.vocab_size,
"device": self.device,
"req_states": self.req_states,
"logprobs_mode": self.model_config.logprobs_mode,
"num_speculative_tokens": self.decode_query_len,
"use_fp64_gumbel": self.model_config.use_fp64_gumbel,
"enable_trace_replay": self.model_config.enable_trace_replay,
"reasoning_config": self.vllm_config.reasoning_config,
"return_sampling_mask": self.model_config.return_sampling_mask,
}
if self.vllm_config.watermark_config is None:
self.sampler = Sampler(**sampler_kwargs)
else:
wm_config = self.vllm_config.watermark_config
watermarker = create_watermarker(wm_config)
if self.speculative_config is not None:
watermarker = create_speculative_target_watermarker(watermarker)
self.sampler = GPUWatermarkSampler(
watermarker,
deduplicate_contexts=wm_config.deduplicate_contexts,
deduplicate_contexts_max_history=(
wm_config.deduplicate_contexts_max_history
),
**sampler_kwargs,
)
custom = self.model_state.custom_sampler(self.sampler)
if custom:
self.vllm_config._check_watermarking_unsupported(custom_sampler=True)
self.sampler, self.rejection_sampler = custom
elif self.speculative_config is not None:
self.rejection_sampler = RejectionSampler(
self.sampler,
self.speculative_config,
self.device,
watermark_key=speculative_target_watermark_key(
self.vllm_config.watermark_config
),
)
if custom:
self.vllm_config._check_watermarking_unsupported(
custom_sampler=True
)
self.sampler, self.rejection_sampler = custom
elif self.speculative_config is not None:
self.rejection_sampler = RejectionSampler(
self.sampler,
self.speculative_config,
self.device,
watermark_key=speculative_target_watermark_key(
self.vllm_config.watermark_config
),
)
self.prompt_logprobs_worker = PromptLogprobsWorker(
self.max_num_reqs,
logprobs_mode=self.model_config.logprobs_mode,
@@ -738,7 +741,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
self.speculator.init_cudagraph_manager(cudagraph_mode)
# Capture warmup providers that depend on allocated KV-cache strides.
with self.jit_warmup_registry.activate():
with self.jit_warmup_registry.activate(): # type: ignore[attr-defined]
kv_caches_dict = init_kv_cache(
self.compilation_config.static_forward_context,
self.kv_cache_config,
@@ -924,6 +927,12 @@ class GPUModelRunner(LoRAModelRunnerMixin):
# during actual execution.
assert self.sampler is not None
self.sampler(logits, dummy_input_batch)
if self.rejection_sampler is not None:
# Warm head-dtype logits; scheduler warmup covers processed FP32 logits.
assert self.speculator is not None
self.rejection_sampler(
logits, dummy_input_batch, self.speculator.draft_logits
)
@torch.inference_mode()
def _dummy_pooler_run(self, hidden_states: torch.Tensor) -> None: