[Neuron] Adding support for context-lenght, token-gen buckets. (#7885)

Co-authored-by: Harsha Bikki <harbikh@amazon.com>
This commit is contained in:
Harsha vardhan manoj Bikki
2024-08-29 13:58:14 -07:00
committed by GitHub
co-authored by Harsha Bikki
parent 86a677de42
commit 257afc37c5
2 changed files with 33 additions and 11 deletions
+9 -2
View File
@@ -1,5 +1,12 @@
import os
from vllm import LLM, SamplingParams
# creates XLA hlo graphs for all the context length buckets.
os.environ['NEURON_CONTEXT_LENGTH_BUCKETS'] = "128,512,1024,2048"
# creates XLA hlo graphs for all the token gen buckets.
os.environ['NEURON_TOKEN_GEN_BUCKETS'] = "128,512,1024,2048"
# Sample prompts.
prompts = [
"Hello, my name is",
@@ -19,8 +26,8 @@ llm = LLM(
# Currently, this is a known limitation in continuous batching support
# in transformers-neuronx.
# TODO(liangfu): Support paged-attention in transformers-neuronx.
max_model_len=128,
block_size=128,
max_model_len=2048,
block_size=2048,
# The device can be automatically detected when AWS Neuron SDK is installed.
# The device argument can be either unspecified for automated detection,
# or explicitly assigned.