Mirrors/text-generation-webui
1
0
Fork 0
mirror of https://github.com/oobabooga/text-generation-webui.git synced 2024-09-20 10:35:10 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Make model autodetect all gpt-neo and opt models

Browse source
This commit is contained in:
oobabooga 2023-01-06 02:31:54 -03:00
parent deefa2e86a
commit 9498dca748
1 changed files with 1 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
server.py
View file

@ -27,7 +27,7 @@ def load_model(model_name):
if os.path.exists(f"torch-dumps/{model_name}.pt"): if os.path.exists(f"torch-dumps/{model_name}.pt"):
print("Loading in .pt format...") print("Loading in .pt format...")
model = torch.load(f"torch-dumps/{model_name}.pt").cuda() model = torch.load(f"torch-dumps/{model_name}.pt").cuda()
elif model_name in ['gpt-neox-20b', 'opt-13b', 'OPT-13B-Erebus']: elif model_name.lower().startswith(('gpt-neo', 'opt-')):
model = AutoModelForCausalLM.from_pretrained(f"models/{model_name}", device_map='auto', load_in_8bit=True) model = AutoModelForCausalLM.from_pretrained(f"models/{model_name}", device_map='auto', load_in_8bit=True)
elif model_name in ['gpt-j-6B']: elif model_name in ['gpt-j-6B']:
model = AutoModelForCausalLM.from_pretrained(f"models/{model_name}", low_cpu_mem_usage=True, torch_dtype=torch.float16).cuda() model = AutoModelForCausalLM.from_pretrained(f"models/{model_name}", low_cpu_mem_usage=True, torch_dtype=torch.float16).cuda()