[Bugfix][CI] fix typos (#34934)
Signed-off-by: 1195343015 <1195343015@qq.com> Signed-off-by: Jiayi Yan <66017932+1195343015@users.noreply.github.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
co-authored by
Harry Mellor
parent
8c760b6ab6
commit
6a895197fa
@@ -905,7 +905,7 @@ class GPUModelRunner(
|
||||
Args:
|
||||
scheduler_output: The scheduler output.
|
||||
"""
|
||||
# Attention free models have zero kv_cache_goups, however models
|
||||
# Attention free models have zero kv_cache_groups, however models
|
||||
# like Mamba are also attention free but use the kv_cache for
|
||||
# keeping its internal state. This is why we check the number
|
||||
# of kv_cache groups instead of solely checking
|
||||
@@ -1065,7 +1065,7 @@ class GPUModelRunner(
|
||||
# of the request. for example:
|
||||
# fist step: num_computed_tokens = 0, spec_tokens = [],
|
||||
# prev_num_draft_len = 0.
|
||||
# second step: num_computed_tokens = 100(prompt lenth),
|
||||
# second step: num_computed_tokens = 100(prompt length),
|
||||
# spec_tokens = [a,b], prev_num_draft_len = 0.
|
||||
# third step: num_computed_tokens = 100 + 2, spec_tokens = [c,d],
|
||||
# prev_num_draft_len = 2.
|
||||
@@ -1412,30 +1412,30 @@ class GPUModelRunner(
|
||||
prev_draft_token_indices.extend(range(start, start + draft_len))
|
||||
indices_match &= prev_index == flattened_index
|
||||
max_flattened_index = max(max_flattened_index, flattened_index)
|
||||
num_commmon_tokens = len(sample_flattened_indices)
|
||||
num_common_tokens = len(sample_flattened_indices)
|
||||
total_without_spec = total_num_scheduled_tokens - total_num_spec_tokens
|
||||
if num_commmon_tokens < total_without_spec:
|
||||
if num_common_tokens < total_without_spec:
|
||||
# If not all requests are decodes from the last iteration,
|
||||
# We need to copy the input_ids_cpu to the GPU first.
|
||||
self.input_ids.copy_to_gpu(total_num_scheduled_tokens)
|
||||
if self.enable_prompt_embeds:
|
||||
self.inputs_embeds.copy_to_gpu(total_num_scheduled_tokens)
|
||||
self.is_token_ids.copy_to_gpu(total_num_scheduled_tokens)
|
||||
if num_commmon_tokens == 0:
|
||||
if num_common_tokens == 0:
|
||||
# No requests in common with the previous iteration
|
||||
# So input_ids.cpu will have all the input ids.
|
||||
return
|
||||
if indices_match and max_flattened_index == (num_commmon_tokens - 1):
|
||||
if indices_match and max_flattened_index == (num_common_tokens - 1):
|
||||
# Common-case optimization: the batch is unchanged
|
||||
# and no reordering happened.
|
||||
# The indices are both the same permutation of 0..N-1 so
|
||||
# we can copy directly using a single slice.
|
||||
self.input_ids.gpu[:num_commmon_tokens].copy_(
|
||||
self.input_batch.prev_sampled_token_ids[:num_commmon_tokens, 0],
|
||||
self.input_ids.gpu[:num_common_tokens].copy_(
|
||||
self.input_batch.prev_sampled_token_ids[:num_common_tokens, 0],
|
||||
non_blocking=True,
|
||||
)
|
||||
if self.enable_prompt_embeds:
|
||||
self.is_token_ids.gpu[:num_commmon_tokens] = True
|
||||
self.is_token_ids.gpu[:num_common_tokens] = True
|
||||
return
|
||||
# Upload the index tensors asynchronously so the scatter can be non-blocking.
|
||||
sampled_tokens_index_tensor = torch.tensor(
|
||||
@@ -4383,7 +4383,7 @@ class GPUModelRunner(
|
||||
self.model.compile(fullgraph=True, backend=backend)
|
||||
return
|
||||
# for other compilation modes, cudagraph behavior is controlled by
|
||||
# CudagraphWraper and CudagraphDispatcher of vllm.
|
||||
# CudagraphWrapper and CudagraphDispatcher of vllm.
|
||||
|
||||
# wrap the model with full cudagraph wrapper if needed.
|
||||
cudagraph_mode = self.compilation_config.cudagraph_mode
|
||||
@@ -4444,7 +4444,7 @@ class GPUModelRunner(
|
||||
:param weights_path: path to load weights from if weights_iterator is not
|
||||
provided. Use path of original model if neither is provided.
|
||||
:param is_checkpoint_format: set to False if weights have already been processed
|
||||
into kernel format (repacking, renaming, ect.)
|
||||
into kernel format (repacking, renaming, etc.)
|
||||
"""
|
||||
# TODO(@kylesayrs): generalize to all runners and loaders
|
||||
# argument validation
|
||||
|
||||
Reference in New Issue
Block a user