zijie-tian/nano-vllm
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix

Browse Source
This commit is contained in:
GeeeekExplorer
2025-06-11 21:17:23 +08:00
parent 08c84ec08d
commit fee58d44e4
5 changed files with 4 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
bench.py
View File

@@ -15,6 +15,6 @@ prompt_token_ids = torch.randint(0, 10240, (batch_size, seq_len)).tolist()
sampling_params = SamplingParams(temperature=0.6, ignore_eos=True, max_tokens=max_tokens)
t = time.time()
completions = llm.generate(prompt_token_ids, sampling_params)
llm.generate(prompt_token_ids, sampling_params)
throughput = batch_size * max_tokens / (time.time() - t)
print(f"Throughput: {throughput: .2f}")