[6/N][warmup][DSv4] Migrate sampling, and DFlash JIT kernels (#56323)

Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Signed-off-by: LopezCastroRoberto <rocastro@redhat.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Roberto L. Castro
2026-09-12 17:24:24 -07:00
committed by GitHub
co-authored by Lucas Wilkinson OpenAI Codex
parent e19a3e172e
commit aed894c190
27 changed files with 2288 additions and 558 deletions
+33 -34
View File
@@ -498,6 +498,7 @@ class ExecuteModelState(NamedTuple):
class GPUModelRunner(
LoRAModelRunnerMixin, KVConnectorModelRunnerMixin, ECConnectorModelRunnerMixin
):
@JitWarmupRegistry.capture
def __init__(
self,
vllm_config: VllmConfig,
@@ -514,7 +515,6 @@ class GPUModelRunner(
self.scheduler_config = vllm_config.scheduler_config
self.speculative_config = vllm_config.speculative_config
self.observability_config = vllm_config.observability_config
self.jit_warmup_registry = JitWarmupRegistry(vllm_config)
model_config = self.model_config
cache_config = self.cache_config
@@ -750,36 +750,35 @@ class GPUModelRunner(
self.parallel_config.cp_kv_cache_interleave_size
)
# Capture warmup providers registered by the initial placeholder InputBatch
with self.jit_warmup_registry.activate():
self.input_batch = InputBatch(
max_num_reqs=self.max_num_reqs,
# We need to use the encoder length for encoder-decoder
# because of KV cache for cross-attention.
max_model_len=max(self.max_model_len, self.max_encoder_len),
max_num_batched_tokens=self.max_num_tokens,
device=self.device,
vocab_size=self.model_config.get_vocab_size(),
block_sizes=[placeholder_block_size],
kernel_block_sizes=[placeholder_block_size],
max_num_blocks_per_req=[placeholder_max_num_blocks],
num_spec_tokens=self.num_spec_tokens,
logitsprocs=build_logitsprocs(
self.vllm_config,
self.device,
PIN_MEMORY,
self.is_pooling_model,
custom_logitsprocs,
),
# We currently don't know whether a particular custom logits processor
# uses output token ids so we set this conservatively. Thinking-budget
# tracking is requested dynamically when a budgeted request is in the
# batch.
logitsprocs_need_output_token_ids=bool(custom_logitsprocs),
is_pooling_model=self.is_pooling_model,
cp_kv_cache_interleave_size=self.parallel_config.cp_kv_cache_interleave_size,
reasoning_config=self.vllm_config.reasoning_config,
use_replayssm=self.cache_config.use_replayssm,
)
self.input_batch = InputBatch(
max_num_reqs=self.max_num_reqs,
# We need to use the encoder length for encoder-decoder
# because of KV cache for cross-attention.
max_model_len=max(self.max_model_len, self.max_encoder_len),
max_num_batched_tokens=self.max_num_tokens,
device=self.device,
vocab_size=self.model_config.get_vocab_size(),
block_sizes=[placeholder_block_size],
kernel_block_sizes=[placeholder_block_size],
max_num_blocks_per_req=[placeholder_max_num_blocks],
num_spec_tokens=self.num_spec_tokens,
logitsprocs=build_logitsprocs(
self.vllm_config,
self.device,
PIN_MEMORY,
self.is_pooling_model,
custom_logitsprocs,
),
# We currently don't know whether a particular custom logits processor
# uses output token ids so we set this conservatively. Thinking-budget
# tracking is requested dynamically when a budgeted request is in the
# batch.
logitsprocs_need_output_token_ids=bool(custom_logitsprocs),
is_pooling_model=self.is_pooling_model,
cp_kv_cache_interleave_size=self.parallel_config.cp_kv_cache_interleave_size,
reasoning_config=self.vllm_config.reasoning_config,
use_replayssm=self.cache_config.use_replayssm,
)
# Separate cuda stream for overlapping transfer of sampled token ids from
# GPU to CPU when async scheduling is enabled.
@@ -5332,7 +5331,7 @@ class GPUModelRunner(
self.load_config.load_format = "dummy"
model_loader = get_model_loader(self.load_config)
# Capture warmup providers selected while constructing the model.
with self.jit_warmup_registry.activate():
with self.jit_warmup_registry.activate(): # type: ignore[attr-defined]
self.model = model_loader.load_model(
vllm_config=self.vllm_config, model_config=self.model_config
)
@@ -7304,7 +7303,7 @@ class GPUModelRunner(
self.parallel_config.cp_kv_cache_interleave_size
)
# Capture warmup providers registered after final KV-cache geometry is known
with self.jit_warmup_registry.activate():
with self.jit_warmup_registry.activate(): # type: ignore[attr-defined]
self.input_batch = InputBatch(
max_num_reqs=self.max_num_reqs,
max_model_len=max_model_len,
@@ -7454,7 +7453,7 @@ class GPUModelRunner(
# Reinitialize need to after initialize_attn_backend
self.may_reinitialize_input_batch(kv_cache_config, kernel_block_sizes)
# Capture warmup providers that depend on allocated KV-cache strides.
with self.jit_warmup_registry.activate():
with self.jit_warmup_registry.activate(): # type: ignore[attr-defined]
kv_caches = self.initialize_kv_cache_tensors(
kv_cache_config,
kernel_block_sizes,