[TPU][V1] Refine tpu_model_runner to mitigate future recompilation issues (#16275)
Signed-off-by: Chengji Yao <chengjiyao@google.com>
This commit is contained in:
@@ -44,7 +44,7 @@ def test_tpu_compilation():
|
||||
assert generated_text.startswith(answer)
|
||||
|
||||
compiled_codes = sorted(
|
||||
glob.glob(os.path.join(temp_dir, "__transformed_code*.py")))
|
||||
glob.glob(os.path.join(temp_dir, "__transformed_code*for_forward.py")))
|
||||
|
||||
for i, compiled_code in enumerate(compiled_codes):
|
||||
print("{} file: {}".format(i + 1, compiled_code))
|
||||
@@ -52,15 +52,21 @@ def test_tpu_compilation():
|
||||
# We should only trigger Dynamo compilation 2 times:
|
||||
# 1. Forward pass without kv_caches
|
||||
# 2. Forward pass with kv_caches
|
||||
# Check we have 4 compiled codes
|
||||
# Check we have 2 compiled codes
|
||||
assert len(compiled_codes) == 2
|
||||
|
||||
kv_cache_prefix = "kv_cache"
|
||||
attn_prefix = "ragged_paged_attention"
|
||||
|
||||
def extract_compiled_index(s):
|
||||
parts = s.replace(".", "_").split("_")
|
||||
numbers = [int(part) for part in parts if part.isdigit()]
|
||||
return numbers[0]
|
||||
|
||||
# Check all the compilations are as expected
|
||||
compiled_fns = sorted(
|
||||
glob.glob(os.path.join(temp_dir, "__compiled_fn*Captured*.py")))
|
||||
compiled_fns = sorted(glob.glob(
|
||||
os.path.join(temp_dir, "__compiled_fn*Captured*.py")),
|
||||
key=lambda s: extract_compiled_index(s))
|
||||
|
||||
for i, compiled_fn in enumerate(compiled_fns):
|
||||
print("{} file: {}".format(i + 1, compiled_fn))
|
||||
|
||||
@@ -7,9 +7,9 @@ from vllm.config import CacheConfig, ModelConfig, SchedulerConfig, VllmConfig
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.v1.core.sched.output import (CachedRequestData, NewRequestData,
|
||||
SchedulerOutput)
|
||||
from vllm.v1.worker.tpu_model_runner import (TPUModelRunner,
|
||||
_get_padded_token_len,
|
||||
_get_paddings)
|
||||
from vllm.v1.worker.tpu_model_runner import (
|
||||
TPUModelRunner, _get_padded_num_reqs_with_upper_limit,
|
||||
_get_padded_token_len, _get_req_paddings, _get_token_paddings)
|
||||
|
||||
# Mock torch_xla module since it may not be available in the test environments
|
||||
torch_xla_patcher = mock.patch.dict(
|
||||
@@ -296,16 +296,29 @@ def test_update_states_request_unscheduled(model_runner):
|
||||
def test_get_paddings():
|
||||
min_token_size, max_token_size, padding_gap = 16, 512, 64
|
||||
expected_paddings = [16, 32, 64, 128, 192, 256, 320, 384, 448, 512]
|
||||
actual_paddings = _get_paddings(min_token_size, max_token_size,
|
||||
padding_gap)
|
||||
actual_paddings = _get_token_paddings(min_token_size, max_token_size,
|
||||
padding_gap)
|
||||
assert actual_paddings == expected_paddings
|
||||
|
||||
|
||||
def test_get_padded_token_len():
|
||||
min_token_size, max_token_size, padding_gap = 16, 512, 64
|
||||
paddings = _get_paddings(min_token_size, max_token_size, padding_gap)
|
||||
paddings = _get_token_paddings(min_token_size, max_token_size, padding_gap)
|
||||
assert _get_padded_token_len(paddings, 1) == 16
|
||||
assert _get_padded_token_len(paddings, 16) == 16
|
||||
assert _get_padded_token_len(paddings, 20) == 32
|
||||
assert _get_padded_token_len(paddings, 300) == 320
|
||||
assert _get_padded_token_len(paddings, 512) == 512
|
||||
|
||||
|
||||
def test_get_padded_num_reqs_with_upper_limit():
|
||||
assert _get_padded_num_reqs_with_upper_limit(3, 32) == 8
|
||||
assert _get_padded_num_reqs_with_upper_limit(9, 32) == 16
|
||||
assert _get_padded_num_reqs_with_upper_limit(19, 32) == 32
|
||||
assert _get_padded_num_reqs_with_upper_limit(17, 28) == 28
|
||||
|
||||
|
||||
def test_get_req_paddings():
|
||||
assert _get_req_paddings(1, 32) == [8, 16, 32]
|
||||
assert _get_req_paddings(8, 32) == [8, 16, 32]
|
||||
assert _get_req_paddings(8, 36) == [8, 16, 32, 36]
|
||||
|
||||
Reference in New Issue
Block a user