[BugFix] Fix use of per-request seed with pipeline parallel (#6698)
This commit is contained in:
@@ -118,6 +118,7 @@ class SamplingMetadata:
|
||||
query_lens: Optional[List[int]],
|
||||
device: str,
|
||||
pin_memory: bool,
|
||||
generators: Optional[Dict[str, torch.Generator]] = None,
|
||||
) -> "SamplingMetadata":
|
||||
(
|
||||
seq_groups,
|
||||
@@ -125,7 +126,7 @@ class SamplingMetadata:
|
||||
categorized_sample_indices,
|
||||
num_prompts,
|
||||
) = _prepare_seq_groups(seq_group_metadata_list, seq_lens, query_lens,
|
||||
device)
|
||||
device, generators)
|
||||
selected_token_indices = async_tensor_h2d(selected_token_indices,
|
||||
dtype=torch.long,
|
||||
target_device=device,
|
||||
@@ -160,6 +161,7 @@ def _prepare_seq_groups(
|
||||
seq_lens: List[int],
|
||||
query_lens: Optional[List[int]],
|
||||
device: str,
|
||||
generators: Optional[Dict[str, torch.Generator]] = None,
|
||||
) -> Tuple[List[SequenceGroupToSample], List[int], Dict[
|
||||
SamplingType, List[Tuple[int, int]]], int]:
|
||||
"""Prepare sequence groups and indices for sampling.
|
||||
@@ -170,8 +172,10 @@ def _prepare_seq_groups(
|
||||
Index of prompt len should match with seq_group_metadata_list.
|
||||
query_lens: A list of query lengths. Prompt lens include the length
|
||||
of entire prompt tokens, and it could be shorter.
|
||||
device: A device to use for random number generator,
|
||||
device: A device to use for random number generators,
|
||||
`SequenceGroupToSample.generator`.
|
||||
generators: A store of per-request random number generators used
|
||||
for seeded requests.
|
||||
|
||||
Returns:
|
||||
seq_groups: A list of sequence group to sample.
|
||||
@@ -217,8 +221,10 @@ def _prepare_seq_groups(
|
||||
|
||||
if seq_group_metadata.is_prompt:
|
||||
if sampling_params.seed is not None:
|
||||
seq_group_metadata.state.generator = torch.Generator(
|
||||
device=device).manual_seed(sampling_params.seed)
|
||||
generator = torch.Generator(device=device).manual_seed(
|
||||
sampling_params.seed)
|
||||
if generators is not None:
|
||||
generators[seq_group_metadata.request_id] = generator
|
||||
|
||||
num_prompts += 1
|
||||
num_prefill_sample = len(seq_ids)
|
||||
@@ -235,6 +241,9 @@ def _prepare_seq_groups(
|
||||
prompt_logprob_len = 0
|
||||
sample_len = len(seq_ids) if do_sample else 0
|
||||
|
||||
if sampling_params.seed is not None and generators is not None:
|
||||
generator = generators.get(seq_group_metadata.request_id)
|
||||
|
||||
# Update indices to select from the model output.
|
||||
"""
|
||||
This blocks computes selected_token_indices which is used in the
|
||||
@@ -279,9 +288,6 @@ def _prepare_seq_groups(
|
||||
logit_idx += sample_len
|
||||
sample_idx += sample_len
|
||||
|
||||
if sampling_params.seed is not None:
|
||||
generator = seq_group_metadata.state.generator
|
||||
|
||||
seq_groups.append(
|
||||
SequenceGroupToSample(
|
||||
seq_ids=seq_ids,
|
||||
|
||||
Reference in New Issue
Block a user