Support beam search & parallel generation (#7)
This commit is contained in:
@@ -15,6 +15,8 @@ parser.add_argument('--num-gpu-blocks', type=int, default=1024, help='number of
|
||||
parser.add_argument('--num-cpu-blocks', type=int, default=32, help='number of CPU blocks (per GPU)')
|
||||
# NOTE(woosuk): If FlashAttention is used, the float data type is not supported.
|
||||
parser.add_argument('--dtype', type=str, default='half', choices=['half', 'float'], help='data type')
|
||||
# TODO(woosuk): Support fine-grained seeds (e.g., seed per request).
|
||||
parser.add_argument('--seed', type=int, default=0, help='random seed')
|
||||
args = parser.parse_args()
|
||||
|
||||
|
||||
@@ -30,6 +32,7 @@ def main():
|
||||
num_gpu_blocks=args.num_gpu_blocks,
|
||||
num_cpu_blocks=args.num_cpu_blocks,
|
||||
dtype=args.dtype,
|
||||
seed=args.seed,
|
||||
)
|
||||
controllers.append(controller)
|
||||
|
||||
@@ -52,18 +55,18 @@ def main():
|
||||
controllers[i].set_next(controllers[i + 1])
|
||||
controllers[-1].set_next(scheduler)
|
||||
|
||||
# Test the following inputs.
|
||||
test_inputs = [
|
||||
'Ion Stoica is a',
|
||||
'UC Berkeley is',
|
||||
'The future of cloud computing is',
|
||||
('Ion Stoica is a', {'n': 4, 'use_beam_search': True, 'temperature': 0.0}),
|
||||
('UC Berkeley is', {'n': 3, 'temperature': 0.8, 'top_p': 0.99}),
|
||||
('The future of cloud computing is', {}), # Use default parameters.
|
||||
]
|
||||
|
||||
# FIXME
|
||||
while True:
|
||||
if test_inputs:
|
||||
frontend.query(test_inputs.pop())
|
||||
text, sampling_params = test_inputs.pop(0)
|
||||
frontend.query(text, **sampling_params)
|
||||
scheduler.step()
|
||||
if not scheduler.pending and not scheduler.running:
|
||||
if not (scheduler.pending or scheduler.running or test_inputs):
|
||||
break
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user