zijie-tian/nano-vllm
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[feat] Optimized with ASYNC offload.

Browse Source
This commit is contained in:
Zijie Tian
2025-12-15 07:21:35 +08:00
parent b8b6478506
commit 91a0f09a24
3 changed files with 93 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
bench_offload.py
View File

@@ -38,8 +38,8 @@ def main():
llm = LLM(
path,
enforce_eager=False,
max_model_len=128 * 1024,
max_num_batched_tokens=128 * 1024,
max_model_len=256 * 1024,
max_num_batched_tokens=256 * 1024,
enable_cpu_offload=True,
num_gpu_blocks=120,
num_prefetch_blocks=4,
@@ -54,12 +54,12 @@ def main():
# bench_prefill(llm, num_seqs=1, input_len=1024)
# bench_prefill(llm, num_seqs=1, input_len=2048)
# bench_prefill(llm, num_seqs=1, input_len=4096)
bench_prefill(llm, num_seqs=1, input_len=16 * 1024)
bench_prefill(llm, num_seqs=1, input_len=128 * 1024)
print("=" * 60)
print("Decode Benchmark (CPU Offload)")
print("=" * 60)
bench_decode(llm, num_seqs=1, input_len=16 * 1024, max_output_len=128)
bench_decode(llm, num_seqs=1, input_len=128 * 1024, max_output_len=128)
# bench_decode(llm, num_seqs=1, input_len=2048, max_output_len=128)