[6/N][warmup][DSv4] Migrate sampling, and DFlash JIT kernels (#56323)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: LopezCastroRoberto <rocastro@redhat.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
Lucas Wilkinson
OpenAI Codex
parent
e19a3e172e
commit
aed894c190
@@ -4,7 +4,10 @@ import numpy as np
|
||||
import torch
|
||||
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.v1.sample.ops.topk_topp_sampler import apply_top_k_top_p
|
||||
from vllm.v1.sample.ops.topk_topp_sampler import (
|
||||
apply_top_k_top_p,
|
||||
register_top_k_top_p_warmups,
|
||||
)
|
||||
from vllm.v1.worker.gpu.buffer_utils import UvaBackedTensor
|
||||
from vllm.v1.worker.gpu.sample.gumbel import apply_temperature
|
||||
from vllm.v1.worker.gpu.sample.min_p import apply_min_p
|
||||
@@ -18,6 +21,7 @@ class SamplingStates:
|
||||
def __init__(self, max_num_reqs: int, vocab_size: int):
|
||||
self.max_num_reqs = max_num_reqs
|
||||
self.vocab_size = vocab_size
|
||||
register_top_k_top_p_warmups()
|
||||
|
||||
self.temperature = UvaBackedTensor(max_num_reqs, dtype=torch.float32)
|
||||
self.top_k = UvaBackedTensor(max_num_reqs, dtype=torch.int32)
|
||||
|
||||
Reference in New Issue
Block a user