[Core] Avoid seq_lens_cpu GPU->CPU sync (#40654)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -227,12 +227,15 @@ def build_attn_metadata(
|
||||
block_tables: Sequence[torch.Tensor],
|
||||
slot_mappings: torch.Tensor,
|
||||
kv_cache_config: KVCacheConfig,
|
||||
seq_lens_cpu_upper_bound: torch.Tensor | None = None,
|
||||
dcp_local_seq_lens: torch.Tensor | None = None,
|
||||
encoder_seq_lens: dict[int, tuple[torch.Tensor, np.ndarray]] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
seq_lens = seq_lens[:num_reqs]
|
||||
if dcp_local_seq_lens is not None:
|
||||
dcp_local_seq_lens = dcp_local_seq_lens[:num_reqs]
|
||||
if seq_lens_cpu_upper_bound is not None:
|
||||
seq_lens_cpu_upper_bound = seq_lens_cpu_upper_bound[:num_reqs]
|
||||
|
||||
attn_metadata: dict[str, Any] = {}
|
||||
num_kv_cache_groups = len(kv_cache_config.kv_cache_groups)
|
||||
@@ -244,6 +247,7 @@ def build_attn_metadata(
|
||||
query_start_loc=query_start_loc_gpu,
|
||||
query_start_loc_cpu=query_start_loc_cpu,
|
||||
seq_lens=seq_lens,
|
||||
seq_lens_cpu_upper_bound=seq_lens_cpu_upper_bound,
|
||||
max_seq_len=max_seq_len,
|
||||
num_reqs=num_reqs,
|
||||
num_actual_tokens=num_tokens,
|
||||
|
||||
@@ -60,6 +60,8 @@ class InputBatch:
|
||||
query_start_loc_np: np.ndarray
|
||||
# [num_reqs]
|
||||
seq_lens: torch.Tensor
|
||||
# [num_reqs] CPU upper bound on seq_lens (see CommonAttentionMetadata).
|
||||
seq_lens_cpu_upper_bound: torch.Tensor
|
||||
# [num_reqs]
|
||||
dcp_local_seq_lens: torch.Tensor | None
|
||||
|
||||
@@ -121,6 +123,8 @@ class InputBatch:
|
||||
logits_indices = query_start_loc[1:] - 1
|
||||
cu_num_logits = torch.arange(num_reqs + 1, device=device, dtype=torch.int32)
|
||||
cu_num_logits_np = np.arange(num_reqs + 1, dtype=np.int32)
|
||||
# Dummy: seq_len == query_len (fresh-prefill shape).
|
||||
seq_lens_cpu_upper_bound = torch.from_numpy(num_scheduled_tokens.copy())
|
||||
return cls(
|
||||
req_ids=req_ids,
|
||||
num_reqs=num_reqs,
|
||||
@@ -136,6 +140,7 @@ class InputBatch:
|
||||
query_start_loc=query_start_loc,
|
||||
query_start_loc_np=query_start_loc_np,
|
||||
seq_lens=seq_lens,
|
||||
seq_lens_cpu_upper_bound=seq_lens_cpu_upper_bound,
|
||||
dcp_local_seq_lens=None,
|
||||
input_ids=input_ids,
|
||||
positions=positions,
|
||||
|
||||
@@ -799,6 +799,15 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
total_num_logits,
|
||||
)
|
||||
|
||||
# CPU upper bound on seq_lens; padded entries left at zero.
|
||||
seq_lens_cpu_upper_bound_np = np.zeros(num_reqs_padded, dtype=np.int32)
|
||||
np.add(
|
||||
self.req_states.num_computed_tokens_np[idx_mapping_np],
|
||||
num_scheduled_tokens,
|
||||
out=seq_lens_cpu_upper_bound_np[:num_reqs],
|
||||
)
|
||||
seq_lens_cpu_upper_bound = torch.from_numpy(seq_lens_cpu_upper_bound_np)
|
||||
|
||||
return InputBatch(
|
||||
req_ids=req_ids,
|
||||
num_reqs=num_reqs,
|
||||
@@ -814,6 +823,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
query_start_loc=query_start_loc,
|
||||
query_start_loc_np=query_start_loc_np,
|
||||
seq_lens=seq_lens,
|
||||
seq_lens_cpu_upper_bound=seq_lens_cpu_upper_bound,
|
||||
dcp_local_seq_lens=dcp_local_seq_lens,
|
||||
input_ids=self.input_buffers.input_ids[:num_tokens_after_padding],
|
||||
positions=self.input_buffers.positions[:num_tokens_after_padding],
|
||||
@@ -927,6 +937,10 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
np.minimum(
|
||||
computed_prefill, self.req_states.prefill_len.np, out=computed_prefill
|
||||
)
|
||||
# Advance the CPU mirror optimistically (assume all scheduled accepted).
|
||||
self.req_states.num_computed_tokens_np[idx_mapping_np] += (
|
||||
input_batch.num_scheduled_tokens
|
||||
)
|
||||
|
||||
@torch.inference_mode()
|
||||
def execute_model(
|
||||
@@ -1297,6 +1311,10 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
np.minimum(
|
||||
computed_prefill, self.req_states.prefill_len.np, out=computed_prefill
|
||||
)
|
||||
# Advance the CPU mirror optimistically (assume all scheduled accepted).
|
||||
self.req_states.num_computed_tokens_np[idx_mapping_np] += (
|
||||
input_batch.num_scheduled_tokens
|
||||
)
|
||||
|
||||
########### EPLB methods start ###########
|
||||
@property
|
||||
|
||||
@@ -173,6 +173,12 @@ class DefaultModelState(ModelState):
|
||||
num_tokens = input_batch.num_tokens
|
||||
query_start_loc_cpu = torch.from_numpy(input_batch.query_start_loc_np)
|
||||
max_query_len = input_batch.num_scheduled_tokens.max().item()
|
||||
seq_lens_cpu_upper_bound = input_batch.seq_lens_cpu_upper_bound
|
||||
if for_capture:
|
||||
# Capture with worst-case max_seq_len so the graph is valid at any replay.
|
||||
max_seq_len = self.max_model_len
|
||||
else:
|
||||
max_seq_len = int(seq_lens_cpu_upper_bound[:num_reqs].max().item())
|
||||
attn_metadata = build_attn_metadata(
|
||||
attn_groups=attn_groups,
|
||||
num_reqs=num_reqs,
|
||||
@@ -181,10 +187,11 @@ class DefaultModelState(ModelState):
|
||||
query_start_loc_cpu=query_start_loc_cpu,
|
||||
max_query_len=max_query_len,
|
||||
seq_lens=input_batch.seq_lens,
|
||||
max_seq_len=self.max_model_len,
|
||||
max_seq_len=max_seq_len,
|
||||
block_tables=block_tables,
|
||||
slot_mappings=slot_mappings,
|
||||
kv_cache_config=kv_cache_config,
|
||||
seq_lens_cpu_upper_bound=seq_lens_cpu_upper_bound,
|
||||
dcp_local_seq_lens=input_batch.dcp_local_seq_lens,
|
||||
)
|
||||
return attn_metadata
|
||||
|
||||
@@ -117,6 +117,11 @@ class WhisperModelState(ModelState):
|
||||
|
||||
query_start_loc_cpu = torch.from_numpy(input_batch.query_start_loc_np)
|
||||
max_query_len = input_batch.num_scheduled_tokens.max().item()
|
||||
seq_lens_cpu_upper_bound = input_batch.seq_lens_cpu_upper_bound
|
||||
if for_capture:
|
||||
max_seq_len = self.max_model_len
|
||||
else:
|
||||
max_seq_len = int(seq_lens_cpu_upper_bound[:num_reqs].max().item())
|
||||
attn_metadata = build_attn_metadata(
|
||||
attn_groups=attn_groups,
|
||||
num_reqs=num_reqs,
|
||||
@@ -125,10 +130,11 @@ class WhisperModelState(ModelState):
|
||||
query_start_loc_cpu=query_start_loc_cpu,
|
||||
max_query_len=max_query_len,
|
||||
seq_lens=input_batch.seq_lens,
|
||||
max_seq_len=self.max_model_len,
|
||||
max_seq_len=max_seq_len,
|
||||
block_tables=block_tables,
|
||||
slot_mappings=slot_mappings,
|
||||
kv_cache_config=kv_cache_config,
|
||||
seq_lens_cpu_upper_bound=seq_lens_cpu_upper_bound,
|
||||
dcp_local_seq_lens=input_batch.dcp_local_seq_lens,
|
||||
encoder_seq_lens=encoder_seq_lens,
|
||||
)
|
||||
|
||||
@@ -57,6 +57,8 @@ class RequestState:
|
||||
self.num_computed_tokens = StagedWriteTensor(
|
||||
self.max_num_reqs, dtype=torch.int32, device=device
|
||||
)
|
||||
# Optimistic CPU mirror of num_computed_tokens (upper bound on GPU value).
|
||||
self.num_computed_tokens_np = np.zeros(self.max_num_reqs, dtype=np.int32)
|
||||
|
||||
# Last sampled tokens.
|
||||
self.last_sampled_tokens = torch.zeros(
|
||||
@@ -100,6 +102,7 @@ class RequestState:
|
||||
self.total_len.stage_write_elem(req_idx, prefill_len)
|
||||
self.all_token_ids.stage_write(req_idx, 0, all_token_ids)
|
||||
self.num_computed_prefill_tokens[req_idx] = num_computed_tokens
|
||||
self.num_computed_tokens_np[req_idx] = num_computed_tokens
|
||||
self.num_computed_tokens.stage_write_elem(req_idx, num_computed_tokens)
|
||||
|
||||
if num_computed_tokens > 0 and num_computed_tokens <= prefill_len:
|
||||
|
||||
Reference in New Issue
Block a user