Mirrors/text-generation-webui
1
0
Fork 0
mirror of https://github.com/oobabooga/text-generation-webui.git synced 2024-09-20 10:35:10 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Better detect EXL2 models

Browse source
This commit is contained in:
oobabooga 2023-09-23 13:04:27 -07:00
parent 895ec9dadb
commit 7a3ca2c68f
2 changed files with 3 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
modules/models_settings.py
View file

@ -76,6 +76,8 @@ def infer_loader(model_name, model_settings):
loader = 'llama.cpp'
elif re.match(r'.*rwkv.*\.pth', model_name.lower()):
loader = 'RWKV'
elif re.match(r'.*exl2', model_name.lower()):
loader = 'ExLlamav2_HF'
else:
loader = 'Transformers'

2
modules/ui_model_menu.py
View file

@ -251,7 +251,7 @@ def download_model_wrapper(repo_id, specific_file, progress=gr.Progress(), retur
def update_truncation_length(current_length, state):
if state['loader'] in ['ExLlama', 'ExLlama_HF']:
if state['loader'].lower().startswith('exllama'):
return state['max_seq_len']
elif state['loader'] in ['llama.cpp', 'llamacpp_HF', 'ctransformers']:
return state['n_ctx']