Mirrors/text-generation-webui
1
0
Fork 0
mirror of https://github.com/oobabooga/text-generation-webui.git synced 2024-09-20 10:35:10 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Bump llama-cpp-python to use LlamaCache

Browse source
This commit is contained in:
oobabooga 2023-04-16 00:53:40 -03:00
parent ac189011cb
commit d2ea925fa5
2 changed files with 4 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
modules/llamacpp_model_alternative.py
View file

@ -6,7 +6,7 @@ Documentation:
https://abetlen.github.io/llama-cpp-python/
'''
from llama_cpp import Llama
from llama_cpp import Llama, LlamaCache
from modules import shared
from modules.callbacks import Iteratorize
@ -27,6 +27,7 @@ class LlamaCppModel:
'n_threads': shared.args.threads or None
}
self.model = Llama(**params)
self.model.set_cache(LlamaCache)
# This is ugly, but the model and the tokenizer are the same object in this library.
return result, result

4
requirements.txt
View file

@ -14,5 +14,5 @@ tqdm
git+https://github.com/huggingface/peft
transformers==4.28.0
bitsandbytes==0.38.1; platform_system != "Windows"
llama-cpp-python==0.1.33; platform_system != "Windows"
https://github.com/abetlen/llama-cpp-python/releases/download/v0.1.33/llama_cpp_python-0.1.33-cp310-cp310-win_amd64.whl; platform_system == "Windows"
llama-cpp-python==0.1.34; platform_system != "Windows"
https://github.com/abetlen/llama-cpp-python/releases/download/v0.1.34/llama_cpp_python-0.1.34-cp310-cp310-win_amd64.whl; platform_system == "Windows"