Mirrors/text-generation-webui
1
0
Fork 0
mirror of https://github.com/oobabooga/text-generation-webui.git synced 2024-09-20 10:35:10 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Generate 8 tokens at a time in streaming mode instead of just 1

Browse source 
This is a performance optimization.
This commit is contained in:
oobabooga 2023-01-25 10:38:26 -03:00
parent 651eb50dd1
commit ebed1dea56
1 changed files with 2 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
server.py
View file

@ -204,8 +204,8 @@ def generate_reply(question, tokens, inference_settings, selected_model, eos_tok
# Generate the reply 1 token at a time
else:
yield formatted_outputs(question, model_name)
preset = preset.replace('max_new_tokens=tokens', 'max_new_tokens=1')
for i in tqdm(range(tokens)):
preset = preset.replace('max_new_tokens=tokens', 'max_new_tokens=8')
for i in tqdm(range(tokens//8+1)):
output = eval(f"model.generate(input_ids, eos_token_id={n}, stopping_criteria=stopping_criteria_list, {preset}){cuda}")
reply = decode(output[0])
if eos_token is not None and reply[-1] == eos_token: