Mirrors/text-generation-webui
1
0
Fork 0
mirror of https://github.com/oobabooga/text-generation-webui.git synced 2024-09-20 18:45:09 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Don't stream at more than 24 fps

Browse source 
This is a performance optimization
This commit is contained in:
oobabooga 2023-05-31 23:41:42 -03:00
parent a160230893
commit b6c407f51d
1 changed files with 12 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
modules/text_generation.py
View file

@ -188,7 +188,19 @@ def _generate_reply(question, state, eos_token=None, stopping_strings=None, is_c
shared.stop_everything = False
clear_torch_cache()
seed = set_manual_seed(state['seed'])
is_stream = state['stream']
last_update = -1
reply = ''
for reply in generate_func(question, original_question, seed, state, eos_token, stopping_strings, is_chat=is_chat):
if is_stream:
cur_time = time.time()
if cur_time - last_update > 0.041666666666666664: # Limit streaming to 24 fps
last_update = cur_time
yield reply
else:
yield reply
if is_stream:
yield reply